├── LICENSE ├── README.md ├── Teaser.PNG ├── VGG_Model ├── .gitignore └── .keep ├── demo_imgs ├── ILSVRC2012_val_00040251.JPEG ├── ILSVRC2012_val_00040252.JPEG ├── ILSVRC2012_val_00040314.JPEG ├── ILSVRC2012_val_00040316.JPEG ├── ILSVRC2012_val_00040318.JPEG ├── ILSVRC2012_val_00040319.JPEG ├── ILSVRC2012_val_00040322.JPEG ├── ILSVRC2012_val_00040323.JPEG ├── ILSVRC2012_val_00040324.JPEG ├── ILSVRC2012_val_00040326.JPEG ├── ILSVRC2012_val_00040333.JPEG ├── ILSVRC2012_val_00040334.JPEG ├── ILSVRC2012_val_00040335.JPEG ├── ILSVRC2012_val_00040337.JPEG ├── ILSVRC2012_val_00040341.JPEG ├── ILSVRC2012_val_00040344.JPEG ├── ILSVRC2012_val_00040348.JPEG ├── ILSVRC2012_val_00040349.JPEG ├── ILSVRC2012_val_00040353.JPEG ├── ILSVRC2012_val_00040354.JPEG ├── ILSVRC2012_val_00040355.JPEG ├── ILSVRC2012_val_00040356.JPEG ├── ILSVRC2012_val_00040357.JPEG ├── ILSVRC2012_val_00040358.JPEG ├── ILSVRC2012_val_00040359.JPEG ├── ILSVRC2012_val_00040360.JPEG ├── ILSVRC2012_val_00040363.JPEG ├── ILSVRC2012_val_00040369.JPEG ├── ILSVRC2012_val_00040371.JPEG ├── ILSVRC2012_val_00040372.JPEG ├── ILSVRC2012_val_00040373.JPEG ├── ILSVRC2012_val_00040375.JPEG ├── ILSVRC2012_val_00040379.JPEG ├── ILSVRC2012_val_00040382.JPEG ├── ILSVRC2012_val_00040387.JPEG ├── ILSVRC2012_val_00040388.JPEG ├── ILSVRC2012_val_00040389.JPEG ├── ILSVRC2012_val_00040390.JPEG ├── ILSVRC2012_val_00040392.JPEG ├── ILSVRC2012_val_00040393.JPEG ├── ILSVRC2012_val_00040394.JPEG ├── ILSVRC2012_val_00040397.JPEG ├── ILSVRC2012_val_00040403.JPEG ├── ILSVRC2012_val_00040780.JPEG ├── ILSVRC2012_val_00040781.JPEG ├── ILSVRC2012_val_00040782.JPEG ├── ILSVRC2012_val_00040785.JPEG ├── ILSVRC2012_val_00040786.JPEG ├── ILSVRC2012_val_00040787.JPEG ├── ILSVRC2012_val_00040788.JPEG ├── ILSVRC2012_val_00040789.JPEG ├── ILSVRC2012_val_00040790.JPEG ├── ILSVRC2012_val_00040792.JPEG ├── ILSVRC2012_val_00040793.JPEG ├── ILSVRC2012_val_00040796.JPEG ├── ILSVRC2012_val_00040799.JPEG ├── ILSVRC2012_val_00040800.JPEG ├── ILSVRC2012_val_00040801.JPEG ├── ILSVRC2012_val_00040802.JPEG ├── ILSVRC2012_val_00040803.JPEG ├── ILSVRC2012_val_00040804.JPEG ├── ILSVRC2012_val_00040806.JPEG ├── ILSVRC2012_val_00040810.JPEG ├── ILSVRC2012_val_00040812.JPEG ├── ILSVRC2012_val_00040813.JPEG ├── ILSVRC2012_val_00040815.JPEG ├── ILSVRC2012_val_00040816.JPEG ├── ILSVRC2012_val_00040819.JPEG ├── ILSVRC2012_val_00040821.JPEG ├── ILSVRC2012_val_00040823.JPEG ├── ILSVRC2012_val_00040825.JPEG ├── ILSVRC2012_val_00040826.JPEG ├── ILSVRC2012_val_00040828.JPEG ├── ILSVRC2012_val_00040829.JPEG ├── ILSVRC2012_val_00040830.JPEG ├── ILSVRC2012_val_00040831.JPEG ├── ILSVRC2012_val_00040833.JPEG ├── ILSVRC2012_val_00040837.JPEG ├── ILSVRC2012_val_00040840.JPEG ├── ILSVRC2012_val_00040843.JPEG ├── ILSVRC2012_val_00040845.JPEG ├── ILSVRC2012_val_00040847.JPEG ├── ILSVRC2012_val_00040850.JPEG ├── ILSVRC2012_val_00043250.JPEG ├── ILSVRC2012_val_00043251.JPEG ├── ILSVRC2012_val_00043252.JPEG ├── ILSVRC2012_val_00043253.JPEG ├── ILSVRC2012_val_00043254.JPEG ├── ILSVRC2012_val_00043255.JPEG ├── ILSVRC2012_val_00043256.JPEG ├── ILSVRC2012_val_00043257.JPEG ├── ILSVRC2012_val_00043258.JPEG ├── ILSVRC2012_val_00043259.JPEG ├── ILSVRC2012_val_00043260.JPEG ├── ILSVRC2012_val_00043261.JPEG ├── ILSVRC2012_val_00043262.JPEG ├── ILSVRC2012_val_00043263.JPEG ├── ILSVRC2012_val_00043264.JPEG ├── ILSVRC2012_val_00043265.JPEG ├── ILSVRC2012_val_00043266.JPEG ├── ILSVRC2012_val_00043268.JPEG ├── ILSVRC2012_val_00043269.JPEG ├── ILSVRC2012_val_00043270.JPEG ├── ILSVRC2012_val_00043271.JPEG ├── ILSVRC2012_val_00043273.JPEG ├── ILSVRC2012_val_00043274.JPEG ├── ILSVRC2012_val_00043275.JPEG ├── ILSVRC2012_val_00043276.JPEG ├── ILSVRC2012_val_00043277.JPEG ├── ILSVRC2012_val_00043278.JPEG ├── ILSVRC2012_val_00043279.JPEG ├── ILSVRC2012_val_00043280.JPEG ├── ILSVRC2012_val_00043281.JPEG ├── ILSVRC2012_val_00043282.JPEG ├── ILSVRC2012_val_00043283.JPEG ├── ILSVRC2012_val_00043284.JPEG ├── ILSVRC2012_val_00043285.JPEG ├── ILSVRC2012_val_00043287.JPEG ├── ILSVRC2012_val_00043288.JPEG ├── ILSVRC2012_val_00043290.JPEG ├── ILSVRC2012_val_00043291.JPEG ├── ILSVRC2012_val_00043292.JPEG ├── ILSVRC2012_val_00043293.JPEG ├── ILSVRC2012_val_00043295.JPEG ├── ILSVRC2012_val_00043296.JPEG ├── ILSVRC2012_val_00043297.JPEG ├── ILSVRC2012_val_00043298.JPEG ├── ILSVRC2012_val_00043300.JPEG ├── ILSVRC2012_val_00043301.JPEG ├── ILSVRC2012_val_00043302.JPEG ├── ILSVRC2012_val_00043303.JPEG ├── ILSVRC2012_val_00043304.JPEG ├── ILSVRC2012_val_00043305.JPEG ├── ILSVRC2012_val_00043306.JPEG ├── ILSVRC2012_val_00043307.JPEG ├── ILSVRC2012_val_00043308.JPEG ├── ILSVRC2012_val_00043311.JPEG ├── ILSVRC2012_val_00043312.JPEG ├── ILSVRC2012_val_00043313.JPEG ├── ILSVRC2012_val_00043314.JPEG ├── ILSVRC2012_val_00043315.JPEG ├── ILSVRC2012_val_00043316.JPEG ├── ILSVRC2012_val_00043317.JPEG ├── ILSVRC2012_val_00043318.JPEG ├── ILSVRC2012_val_00043320.JPEG ├── ILSVRC2012_val_00043321.JPEG ├── ILSVRC2012_val_00043322.JPEG ├── ILSVRC2012_val_00043323.JPEG ├── ILSVRC2012_val_00043325.JPEG ├── ILSVRC2012_val_00043327.JPEG ├── ILSVRC2012_val_00043328.JPEG ├── ILSVRC2012_val_00043329.JPEG ├── ILSVRC2012_val_00043330.JPEG ├── ILSVRC2012_val_00043331.JPEG ├── ILSVRC2012_val_00043333.JPEG ├── ILSVRC2012_val_00043334.JPEG ├── ILSVRC2012_val_00043335.JPEG ├── ILSVRC2012_val_00043336.JPEG ├── ILSVRC2012_val_00043337.JPEG ├── ILSVRC2012_val_00043338.JPEG ├── ILSVRC2012_val_00043339.JPEG ├── ILSVRC2012_val_00043340.JPEG ├── ILSVRC2012_val_00043341.JPEG ├── ILSVRC2012_val_00043342.JPEG ├── ILSVRC2012_val_00043343.JPEG ├── ILSVRC2012_val_00043344.JPEG ├── ILSVRC2012_val_00043345.JPEG ├── ILSVRC2012_val_00043346.JPEG ├── ILSVRC2012_val_00043355.JPEG ├── ILSVRC2012_val_00043357.JPEG ├── ILSVRC2012_val_00043358.JPEG ├── ILSVRC2012_val_00043363.JPEG ├── ILSVRC2012_val_00043364.JPEG ├── ILSVRC2012_val_00043365.JPEG └── ILSVRC2012_val_00043366.JPEG ├── demo_vid ├── frame_000980.jpg ├── frame_001010.jpg ├── frame_001040.jpg └── frame_001070.jpg ├── download_pretrained.sh ├── environment.yml ├── flow_warp.py ├── lib ├── flow_warp.so └── flow_warp_new.so ├── loss.py ├── main_whole.py ├── myflowlib.py ├── network.py ├── test.py ├── test_div_video.py ├── utils.py └── video_utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Chenyang LEI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | This is a Tensorflow implementation for the CVPR 2019 paper 'Fully Automatic Video Colorization with Self-Regularization and Diversity'. 3 | 4 | 5 | ![alt text](https://github.com/ChenyangLEI/Fully-Automatic-Video-Colorization-with-Self-Regularization-and-Diversity/blob/master/Teaser.PNG) 6 | 7 | More results are shown on our project website https://leichenyang.weebly.com/project-color.html 8 | 9 | ## News 10 | We propose a novel and general framework [Deep-Video-Prior](https://chenyanglei.github.io/DVP/index.html) 11 | that can address the temporal inconsistency problem given an input video and a processed video. 12 | We can obtain high-quality video using a single-image colorization method and our novel framework. 13 | 14 | 15 | 16 | ## Quick inference( without refinement network) 17 | ``` 18 | conda env create -f environment.yml 19 | conda activate automatic-video-colorization 20 | bash pretrained_models/download_models.sh 21 | python test.py 22 | ``` 23 | the results are saved in test_result/ 24 | 25 | ## Dependency 26 | ### Environment 27 | This code is based on tensorflow. It has been tested on Ubuntu 18.04 LTS. 28 | 29 | Anaconda is recommended: [Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-install-the-anaconda-python-distribution-on-ubuntu-18-04) 30 | | [Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-the-anaconda-python-distribution-on-ubuntu-16-04) 31 | 32 | After installing Anaconda, you can setup the environment simply by 33 | 34 | ``` 35 | conda env create -f environment.yml 36 | conda activate automatic-video-colorization 37 | ``` 38 | 39 | ### Pretrained-models and VGG-Models 40 | ``` 41 | bash download_pretrained.sh 42 | ``` 43 | 44 | 45 | ## Usage 46 | ### Image colorization 47 | We provide the ckpt to colorize a single image, the temporal consistency is not as good as video colorization but the colorization performance is better. 48 | 49 | You can colorization a single image, e.g.: 50 | ``` 51 | python test.py --img_path PATH/TO/IMAGE 52 | e.g., 53 | python test.py --img_path demo_imgs/ILSVRC2012_val_00040251.JPEG 54 | ``` 55 | 56 | or colorize the images in a folder by: 57 | ``` 58 | python test.py --img_path PATH/TO/FOLDER 59 | e.g., 60 | python test.py --img_path demo_imgs/ 61 | ``` 62 | The results are saved in test_result/ 63 | 64 | ### Video colorization without optical flow 65 | Step1. the video should be split to frames first: 66 | ``` 67 | python video_utils.py --video2frames --video_dir demo.mkv --out_frames_dir demo_framesdir 68 | ``` 69 | 70 | Step2. Colorizing blanc-and-white frames (you can also use the image colorization pretrained model to colorize the frames): 71 | ``` 72 | python test_div_video.py --use_gpu 1 --video_path demo_framesdir 73 | ``` 74 | Results are saved in video_path_colorized, e.g., demo_framesdir_colorized 75 | 76 | Step3. Convert colorized frames to video, note that you need to check the fps of original video 77 | ``` 78 | python video_utils.py --frames2video --colorized_video_dir demo_colorized.mp4 --colorized_frames_dir demo_framesdir_colorized --fps 24 79 | ``` 80 | 81 | Step4. Add the sound of original video (again, you need to make sure the fps of colorized vide is consistent with the original video) 82 | ``` 83 | python video_utils.py --add_sound --colorized_video_dir demo_colorized.mp4 --video_dir demo.mkv 84 | ``` 85 | 86 | 87 | 88 | ## Training 89 | 90 | ``` 91 | python main_whole.py --model YOUR_MODEL_NAME --data_dir data 92 | ``` 93 | 94 | ### Prepare the dataset 95 | For the video dataset, please download the DAVIS dataset and generate the optical flow by PWC-Net by yourself. If you want to use FlowNet2 or other methods, please make sure the file format is the same. 96 | 97 | At last, please arrange your data in the following format: 98 | 99 | ``` 100 | +data 101 | -----+JPEGImages 102 | ----------------+480p 103 | ---------------------+VideoFrames1 104 | ---------------------+VideoFrames2 105 | -----+FLOWImages 106 | ----------------+Forward 107 | -----------------------+VideoFrames1 108 | -----------------------+VideoFrames2 109 | ----------------+Backward 110 | -----------------------+VideoFrames1 111 | -----------------------+VideoFrames2 112 | -----+FLOWImages_GRAY 113 | --------------------+Forward 114 | ----------------------------+VideoFrames1 115 | ----------------------------+VideoFrames2 116 | --------------------+Backward 117 | ----------------------------+VideoFrames1 118 | ----------------------------+VideoFrames2 119 | ``` 120 | 121 | For the image dataset, please download the ImageNet dataset. 122 | 123 | ## Citation 124 | If you use our code or paper, please cite: 125 | 126 | ``` 127 | @InProceedings{Lei_2019_CVPR, 128 | author = {Lei, Chenyang and Chen, Qifeng}, 129 | title = {Fully Automatic Video Colorization With Self-Regularization and Diversity}, 130 | booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 131 | month = {June}, 132 | year = {2019} 133 | } 134 | ``` 135 | 136 | ## Contact 137 | If you have any question, please feel free to contact me (Chenyang LEI, leichenyang7@gmail.com) 138 | 139 | -------------------------------------------------------------------------------- /Teaser.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/Teaser.PNG -------------------------------------------------------------------------------- /VGG_Model/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/VGG_Model/.gitignore -------------------------------------------------------------------------------- /VGG_Model/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/VGG_Model/.keep -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040251.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040251.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040252.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040252.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040314.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040314.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040316.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040316.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040318.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040318.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040319.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040319.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040322.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040322.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040323.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040323.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040324.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040324.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040326.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040326.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040333.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040333.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040334.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040334.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040335.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040335.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040337.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040337.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040341.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040341.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040344.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040344.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040348.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040348.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040349.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040349.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040353.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040353.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040354.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040354.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040355.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040355.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040356.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040356.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040357.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040357.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040358.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040358.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040359.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040359.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040360.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040360.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040363.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040363.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040369.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040369.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040371.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040371.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040372.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040372.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040373.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040373.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040375.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040375.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040379.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040379.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040382.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040382.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040387.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040387.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040388.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040388.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040389.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040389.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040390.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040390.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040392.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040392.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040393.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040393.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040394.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040394.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040397.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040397.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040403.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040403.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040780.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040780.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040781.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040781.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040782.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040782.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040785.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040785.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040786.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040786.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040787.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040787.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040788.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040788.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040789.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040789.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040790.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040790.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040792.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040792.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040793.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040793.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040796.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040796.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040799.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040799.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040800.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040800.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040801.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040801.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040802.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040802.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040803.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040803.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040804.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040804.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040806.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040806.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040810.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040810.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040812.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040812.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040813.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040813.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040815.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040815.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040816.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040816.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040819.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040819.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040821.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040821.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040823.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040823.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040825.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040825.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040826.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040826.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040828.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040828.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040829.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040829.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040830.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040830.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040831.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040831.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040833.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040833.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040837.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040837.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040840.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040840.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040843.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040843.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040845.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040845.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040847.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040847.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00040850.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00040850.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043250.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043250.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043251.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043251.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043252.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043252.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043253.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043253.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043254.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043254.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043255.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043255.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043256.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043256.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043257.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043257.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043258.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043258.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043259.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043259.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043260.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043260.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043261.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043261.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043262.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043262.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043263.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043263.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043264.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043264.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043265.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043265.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043266.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043266.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043268.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043268.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043269.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043269.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043270.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043270.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043271.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043271.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043273.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043273.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043274.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043274.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043275.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043275.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043276.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043276.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043277.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043277.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043278.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043278.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043279.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043279.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043280.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043280.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043281.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043281.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043282.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043282.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043283.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043283.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043284.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043284.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043285.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043285.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043287.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043287.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043288.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043288.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043290.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043290.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043291.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043291.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043292.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043292.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043293.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043293.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043295.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043295.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043296.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043296.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043297.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043297.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043298.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043298.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043300.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043300.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043301.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043301.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043302.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043302.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043303.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043303.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043304.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043304.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043305.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043305.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043306.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043306.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043307.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043307.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043308.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043308.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043311.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043311.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043312.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043312.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043313.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043313.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043314.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043314.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043315.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043315.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043316.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043316.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043317.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043317.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043318.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043318.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043320.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043320.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043321.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043321.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043322.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043322.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043323.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043323.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043325.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043325.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043327.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043327.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043328.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043328.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043329.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043329.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043330.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043330.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043331.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043331.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043333.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043333.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043334.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043334.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043335.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043335.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043336.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043336.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043337.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043337.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043338.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043338.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043339.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043339.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043340.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043340.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043341.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043341.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043342.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043342.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043343.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043343.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043344.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043344.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043345.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043345.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043346.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043346.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043355.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043355.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043357.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043357.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043358.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043358.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043363.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043363.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043364.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043364.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043365.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043365.JPEG -------------------------------------------------------------------------------- /demo_imgs/ILSVRC2012_val_00043366.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_imgs/ILSVRC2012_val_00043366.JPEG -------------------------------------------------------------------------------- /demo_vid/frame_000980.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_vid/frame_000980.jpg -------------------------------------------------------------------------------- /demo_vid/frame_001010.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_vid/frame_001010.jpg -------------------------------------------------------------------------------- /demo_vid/frame_001040.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_vid/frame_001040.jpg -------------------------------------------------------------------------------- /demo_vid/frame_001070.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/demo_vid/frame_001070.jpg -------------------------------------------------------------------------------- /download_pretrained.sh: -------------------------------------------------------------------------------- 1 | wget "https://hkustconnect-my.sharepoint.com/:u:/g/personal/cleiaa_connect_ust_hk/EX1uICZZJghHidfgeL8NdDMB5Pz9xtp7wXZ7PFGuGYII1Q?e=y35LiY&download=1" 2 | mv EX1uICZZJghHidfgeL8NdDMB5Pz9xtp7wXZ7PFGuGYII1Q\?e\=y35LiY\&download\=1 pretrained_models.zip 3 | unzip pretrained_models.zip 4 | 5 | wget "https://hkustconnect-my.sharepoint.com/:u:/g/personal/cleiaa_connect_ust_hk/EZeGsvuqh1dJr0E2Fxf6IKoBQ7wZpGi3NFqZxhzC8-3GHg?e=LLWUZT&download=1" 6 | mv EZeGsvuqh1dJr0E2Fxf6IKoBQ7wZpGi3NFqZxhzC8-3GHg\?e\=LLWUZT\&download\=1 VGG_Model.zip 7 | unzip VGG_Model.zip 8 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: automatic-video-colorization 2 | channels: 3 | - defaults 4 | dependencies: 5 | - _libgcc_mutex=0.1=main 6 | - _tflow_select=2.1.0=gpu 7 | - absl-py=0.11.0=py36h06a4308_0 8 | - astor=0.8.1=py36_0 9 | - blas=1.0=mkl 10 | - c-ares=1.16.1=h7b6447c_0 11 | - ca-certificates=2020.10.14=0 12 | - certifi=2020.6.20=pyhd3eb1b0_3 13 | - cudatoolkit=10.0.130=0 14 | - cudnn=7.6.5=cuda10.0_0 15 | - cupti=10.0.130=0 16 | - cycler=0.10.0=py36_0 17 | - dbus=1.13.18=hb2f20db_0 18 | - expat=2.2.10=he6710b0_2 19 | - fontconfig=2.13.0=h9420a91_0 20 | - freetype=2.10.4=h5ab3b9f_0 21 | - gast=0.4.0=py_0 22 | - glib=2.66.1=h92f7085_0 23 | - grpcio=1.31.0=py36hf8bcb03_0 24 | - gst-plugins-base=1.14.0=hbbd80ab_1 25 | - gstreamer=1.14.0=hb31296c_0 26 | - h5py=2.10.0=py36hd6299e0_1 27 | - hdf5=1.10.6=hb1b8bf9_0 28 | - icu=58.2=he6710b0_3 29 | - imageio=2.9.0=py_0 30 | - importlib-metadata=2.0.0=py_1 31 | - intel-openmp=2020.2=254 32 | - jpeg=9b=h024ee3a_2 33 | - keras-applications=1.0.8=py_1 34 | - keras-preprocessing=1.1.0=py_1 35 | - kiwisolver=1.3.0=py36h2531618_0 36 | - lcms2=2.11=h396b838_0 37 | - ld_impl_linux-64=2.33.1=h53a641e_7 38 | - libedit=3.1.20191231=h14c3975_1 39 | - libffi=3.3=he6710b0_2 40 | - libgcc-ng=9.1.0=hdf63c60_0 41 | - libgfortran-ng=7.3.0=hdf63c60_0 42 | - libpng=1.6.37=hbc83047_0 43 | - libprotobuf=3.13.0.1=hd408876_0 44 | - libstdcxx-ng=9.1.0=hdf63c60_0 45 | - libtiff=4.1.0=h2733197_1 46 | - libuuid=1.0.3=h1bed415_2 47 | - libxcb=1.14=h7b6447c_0 48 | - libxml2=2.9.10=hb55368b_3 49 | - lz4-c=1.9.2=heb0550a_3 50 | - markdown=3.3.3=py36h06a4308_0 51 | - matplotlib=3.3.2=0 52 | - matplotlib-base=3.3.2=py36h817c723_0 53 | - mkl=2020.2=256 54 | - mkl-service=2.3.0=py36he904b0f_0 55 | - mkl_fft=1.2.0=py36h23d657b_0 56 | - mkl_random=1.1.1=py36h0573a6f_0 57 | - mock=4.0.2=py_0 58 | - ncurses=6.2=he6710b0_1 59 | - numpy=1.19.2=py36h54aff64_0 60 | - numpy-base=1.19.2=py36hfa32c7d_0 61 | - olefile=0.46=py_0 62 | - openssl=1.1.1h=h7b6447c_0 63 | - pcre=8.44=he6710b0_0 64 | - pillow=8.0.1=py36he98fc37_0 65 | - pip=20.2.4=py36h06a4308_0 66 | - protobuf=3.13.0.1=py36he6710b0_1 67 | - pyparsing=2.4.7=py_0 68 | - pyqt=5.9.2=py36h05f1152_2 69 | - python=3.6.12=hcff3b4d_2 70 | - python-dateutil=2.8.1=py_0 71 | - qt=5.9.7=h5867ecd_1 72 | - readline=8.0=h7b6447c_0 73 | - scipy=1.5.2=py36h0b6359f_0 74 | - setuptools=50.3.1=py36h06a4308_1 75 | - sip=4.19.8=py36hf484d3e_0 76 | - six=1.15.0=py36h06a4308_0 77 | - sqlite=3.33.0=h62c20be_0 78 | - tensorboard=1.13.1=py36hf484d3e_0 79 | - tensorflow=1.13.1=gpu_py36h3991807_0 80 | - tensorflow-base=1.13.1=gpu_py36h8d69cac_0 81 | - tensorflow-estimator=1.13.0=py_0 82 | - tensorflow-gpu=1.13.1=h0d30ee6_0 83 | - termcolor=1.1.0=py36_1 84 | - tk=8.6.10=hbc83047_0 85 | - tornado=6.0.4=py36h7b6447c_1 86 | - werkzeug=1.0.1=py_0 87 | - wheel=0.35.1=pyhd3eb1b0_0 88 | - xz=5.2.5=h7b6447c_0 89 | - zipp=3.4.0=pyhd3eb1b0_0 90 | - zlib=1.2.11=h7b6447c_3 91 | - zstd=1.4.5=h9ceee32_0 92 | - pip: 93 | - chardet==3.0.4 94 | - decorator==4.4.2 95 | - idna==2.10 96 | - imageio-ffmpeg==0.4.2 97 | - moviepy==1.0.3 98 | - opencv-contrib-python==4.4.0.46 99 | - proglog==0.1.9 100 | - pypng==0.0.20 101 | - requests==2.25.0 102 | - tqdm==4.53.0 103 | - urllib3==1.26.2 104 | prefix: /home/chenyang/anaconda3/envs/fa-video-colorization 105 | 106 | -------------------------------------------------------------------------------- /flow_warp.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | _flow_warp_ops = tf.load_op_library( 4 | tf.resource_loader.get_path_to_datafile("./lib/flow_warp.so")) 5 | 6 | 7 | def flow_warp(image, flow): 8 | return _flow_warp_ops.flow_warp(image, flow) 9 | 10 | 11 | @tf.RegisterGradient("FlowWarp") 12 | def _flow_warp_grad(flow_warp_op, gradients): 13 | return _flow_warp_ops.flow_warp_grad(flow_warp_op.inputs[0], 14 | flow_warp_op.inputs[1], 15 | gradients) 16 | -------------------------------------------------------------------------------- /lib/flow_warp.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/lib/flow_warp.so -------------------------------------------------------------------------------- /lib/flow_warp_new.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChenyangLEI/automatic-video-colorization/e872b07d7fdadf6db33c58a570af6d7f43e5a41c/lib/flow_warp_new.so -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | #tensorflow 1.2.0 is needed 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | import os 6 | import time 7 | import cv2 8 | from scipy import io 9 | import tensorflow as tf 10 | import utils as utils 11 | 12 | def smoothL1_loss(x, y, sigma=0.05): 13 | x_y = tf.abs(x - y) 14 | less_mask = tf.less(x_y, sigma) 15 | greater_mask = tf.greater_equal(x_y, sigma) 16 | loss = 0.5 * tf.reduce_mean(tf.square(tf.boolean_mask(x_y, less_mask))) + sigma * tf.reduce_mean(tf.boolean_mask(x_y - 0.5*sigma, greater_mask)) 17 | return loss 18 | 19 | def compute_error(real,fake): 20 | return tf.reduce_mean(tf.abs(fake-real)) 21 | 22 | def Lp_loss(x, y): 23 | vgg_real = utils.build_vgg19(x * 255.0) 24 | vgg_fake = utils.build_vgg19(y * 255.0,reuse=True) 25 | p0=compute_error(vgg_real['input'] , vgg_fake['input'] ) 26 | p1=compute_error(vgg_real['conv1_2'] , vgg_fake['conv1_2'] ) / 2.6 27 | p2=compute_error(vgg_real['conv2_2'] , vgg_fake['conv2_2'] ) / 4.8 28 | p3=compute_error(vgg_real['conv3_2'] , vgg_fake['conv3_2'] ) / 3.7 29 | p4=compute_error(vgg_real['conv4_2'] , vgg_fake['conv4_2']) / 5.6 30 | p5=compute_error(vgg_real['conv5_2'] , vgg_fake['conv5_2']) * 10 / 1.5 31 | return p0+p1+p2+p3+p4+p5 32 | 33 | def RankDiverse_loss(x, y, num): 34 | loss = [] 35 | for i in range(num): 36 | loss.append(Lp_loss(x[:,:,:,3*i:3*i+3], y[:,:,:,3*i:3*i+3])) 37 | return tf.reduce_min(loss) + tf.reduce_sum([0.01*pow(2, num-i)*loss[i] for i in range(num)]) 38 | 39 | def L1_loss(x, y): 40 | return tf.reduce_mean(tf.abs(x-y)) 41 | 42 | def KNN_loss(out, KNN_idxs): 43 | out = tf.reshape(out, [-1,3]) 44 | loss = [] 45 | for i in range(KNN_idxs.get_shape()[-1]): 46 | loss.append(L1_loss(out, tf.gather_nd(out, KNN_idxs[:,i:i+1]))) 47 | return tf.reduce_mean(loss) 48 | -------------------------------------------------------------------------------- /main_whole.py: -------------------------------------------------------------------------------- 1 | #tensorflow 1.2.0 is needed 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | import os,time,cv2,scipy.io 6 | import tensorflow as tf 7 | import tensorflow.contrib.slim as slim 8 | import numpy as np 9 | import utils as utils 10 | import myflowlib as flowlib 11 | import flow_warp as flow_warp_op 12 | import scipy.misc as sic 13 | import subprocess 14 | import network as net 15 | import loss as loss 16 | import argparse 17 | from sklearn.neighbors import NearestNeighbors 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("--model", default='Result_whole', type=str, help="Model Name") 21 | parser.add_argument("--div_num", default=4, type=int, help="diverse num") 22 | parser.add_argument("--save_freq", default=1, type=int, help="save frequency") 23 | parser.add_argument("--test_dir", default='./data/test/JPEGImages/480p/cows', type=str, help="Test dir path") 24 | parser.add_argument("--train_root", default="./data/train/JPEGImages/480p/", type=str, help="Test dir path") 25 | parser.add_argument("--test_root", default="./data/test/JPEGImages/480p/", type=str, help="Test dir path") 26 | parser.add_argument("--imgs_dir", default='../data/Imagenet', type=str, help="Test dir path") 27 | parser.add_argument("--is_training", default=1, type=int, help="Training or test") 28 | parser.add_argument("--continue_training", default=1, type=int, help="Restore checkpoint") 29 | 30 | 31 | ARGS = parser.parse_args() 32 | print(ARGS) 33 | 34 | model=ARGS.model 35 | div_num=ARGS.div_num 36 | save_freq = ARGS.save_freq 37 | test_dir = ARGS.test_dir 38 | train_root = [ARGS.train_root] 39 | test_root = [ARGS.test_root] 40 | is_training=ARGS.is_training 41 | continue_training=ARGS.continue_training 42 | imgs_dir = ARGS.imgs_dir 43 | num_frame = 2 # number of read in frames 44 | 45 | 46 | os.environ["CUDA_VISIBLE_DEVICES"]=str(np.argmax( [int(x.split()[2]) for x in subprocess.Popen("nvidia-smi -q -d Memory | grep -A4 GPU | grep Free", shell=True, stdout=subprocess.PIPE).stdout.readlines()])) 47 | 48 | 49 | def occlusion_mask(im0, im1, flow10): 50 | warp_im0 = flow_warp_op.flow_warp(im0, flow10) 51 | diff = tf.abs(im1 - warp_im0) 52 | mask = tf.reduce_max(diff, axis=3, keep_dims=True) 53 | mask = tf.less(mask, 0.05) 54 | mask = tf.cast(mask, tf.float32) 55 | mask = tf.tile(mask, [1,1,1,3]) 56 | return mask, warp_im0 57 | 58 | def Bilateral_NN(color_image, neigh): 59 | h,w = color_image.shape[:2] 60 | color_image_X = np.tile(np.arange(w),[h,1])/500. 61 | color_image_Y = np.tile(np.arange(h),[w,1]).T/500. 62 | color_image_all = np.concatenate([color_image,color_image_X[:,:,np.newaxis], color_image_Y[:,:,np.newaxis]],axis=2) 63 | neigh.fit(np.reshape(color_image_all, [-1, 5])) 64 | idxs = neigh.kneighbors(np.reshape(color_image_all, [-1, 5]), 5, return_distance=False) 65 | # idxs = np.zeros([h*w, 5],dtype="int32") 66 | return idxs 67 | 68 | def prepare_input_w_flow(path, num_frames,gray=False): 69 | file_id=os.path.splitext(os.path.basename(path))[0] 70 | input_image_src, input_image_target = utils.read_image_sequence(path, num_frames=num_frame) 71 | if input_image_target is None: 72 | return None, None, None, None 73 | if not gray: 74 | input_flow_forward,input_flow_backward = utils.read_flow_sequence(path.replace("JPEGImages","FLOWImages"), num_frames=num_frame) 75 | else: 76 | input_flow_forward,input_flow_backward = utils.read_flow_sequence(path.replace("JPEGImages","FLOWImages_GRAY"), num_frames=num_frame) 77 | h=input_image_src.shape[0]//32*32 78 | w=input_image_src.shape[1]//32*32 79 | return np.float32(np.expand_dims(input_image_src[:h:2,:w:2,:],axis=0)),\ 80 | np.float32(np.expand_dims(input_image_target[:h:2,:w:2,:],axis=0)),\ 81 | np.expand_dims(input_flow_forward[:h:2,:w:2,:],axis=0)/2.0,\ 82 | np.expand_dims(input_flow_backward[:h:2,:w:2,:],axis=0)/2.0 83 | 84 | config=tf.ConfigProto() 85 | config.gpu_options.allow_growth=True 86 | sess=tf.Session(config=config) 87 | train_low=utils.read_image_path(train_root) 88 | test_low=utils.read_image_path(test_root) 89 | 90 | 91 | input_idx=tf.placeholder(tf.int32,shape=[None,5*num_frame]) 92 | input_i=tf.placeholder(tf.float32,shape=[None,None,None,1*num_frame]) 93 | input_target=tf.placeholder(tf.float32,shape=[None,None,None,3*num_frame]) 94 | input_flow_forward=tf.placeholder(tf.float32,shape=[None,None,None,2*(num_frame-1)]) 95 | input_flow_backward=tf.placeholder(tf.float32,shape=[None,None,None,2*(num_frame-1)]) 96 | 97 | 98 | gray_flow_forward=tf.placeholder(tf.float32,shape=[None,None,None,2*(num_frame-1)]) 99 | gray_flow_backward=tf.placeholder(tf.float32,shape=[None,None,None,2*(num_frame-1)]) 100 | c0=tf.placeholder(tf.float32,shape=[None,None,None,3]) 101 | c1=tf.placeholder(tf.float32,shape=[None,None,None,3]) 102 | 103 | 104 | lossDict = {} 105 | objDict={} 106 | 107 | # X0, X1: Gray frames 108 | # Y0, Y1: Ground truth color frames 109 | # C0, C1: Colorized frames 110 | with tf.variable_scope(tf.get_variable_scope()): 111 | X0, X1 = input_i[:,:,:,0:1], input_i[:,:,:,1:2] 112 | Y0, Y1 = input_target[:,:,:,0:3], input_target[:,:,:,3:6] 113 | with tf.variable_scope('individual'): 114 | C0=net.VCN(utils.build(tf.tile(X0, [1,1,1,3])),reuse=False) 115 | C1=net.VCN(utils.build(tf.tile(X1, [1,1,1,3])),reuse=True) 116 | 117 | objDict["mask"],_=occlusion_mask(Y0,Y1,input_flow_backward[:,:,:,0:2]) 118 | objDict["warped"]=flow_warp_op.flow_warp(C0,input_flow_backward[:,:,:,0:2]) 119 | 120 | lossDict["RankDiv_im1"]=loss.RankDiverse_loss(C0, tf.tile(input_target[:,:,:,0:3], [1,1,1,div_num]),div_num) 121 | lossDict["RankDiv_im2"]=loss.RankDiverse_loss(C1, tf.tile(input_target[:,:,:,3:6], [1,1,1,div_num]),div_num) 122 | lossDict["RankDiv"]=lossDict["RankDiv_im1"]+lossDict["RankDiv_im2"] 123 | 124 | lossDict['Bilateral_im1']= sum([loss.KNN_loss(C0[:,:,:,3*i:3*i+3], input_idx[:,0:5]) for i in range(4)]) 125 | lossDict['Bilateral_im2']= sum([loss.KNN_loss(C1[:,:,:,3*i:3*i+3], input_idx[:,5:10])for i in range(4)]) 126 | lossDict['Bilateral']= lossDict['Bilateral_im2'] + lossDict['Bilateral_im1'] 127 | 128 | lossDict["temporal"]=tf.reduce_mean(tf.multiply(tf.abs(objDict["warped"]-C1),tf.tile(objDict["mask"],[1,1,1,4])))*5 129 | 130 | 131 | lossDict["total"]=lossDict["RankDiv"]+lossDict["temporal"]#+lossDict['Bilateral'] 132 | 133 | objDict["prediction_0"]=tf.concat([C0[:,:,:,0:3],C0[:,:,:,9:12],C0[:,:,:,3:6],C0[:,:,:,6:9]],axis=2) 134 | objDict["prediction_1"]=tf.concat([C1[:,:,:,0:3],C1[:,:,:,9:12],C1[:,:,:,3:6],C1[:,:,:,6:9]],axis=2) 135 | 136 | 137 | #-------------RefineNet---------------# 138 | cmap_C, warp_C0 = occlusion_mask(c0, c1, gray_flow_backward[:,:,:,0:2]) 139 | cmap_X, warp_X0 = occlusion_mask(tf.tile(input_i[:,:,:,0:1], [1,1,1,3]), tf.tile(input_i[:,:,:,1:2],[1,1,1,3]),gray_flow_backward[:,:,:,0:2]) 140 | low_conf_mask = tf.cast(tf.greater(cmap_X - cmap_C, 0), tf.float32) 141 | 142 | coarse_C1 = c1*(-low_conf_mask+1) + tf.tile(input_i[:,:,:,1:2],[1,1,1,3])*low_conf_mask 143 | ref_input = tf.concat([coarse_C1, warp_C0, c1, low_conf_mask, cmap_C, cmap_X], axis=3) 144 | # ref_input = tf.concat([warp_C0, c1, cmap_C, cmap_X, low_conf_mask], axis=3) 145 | final_r1 = net.VCRN(ref_input) 146 | # temporal_g_loss = tf.reduce_mean(tf.abs(input_target[:,:,:,3:6]-final_r1)) + tf.reduce_mean(tf.multiply(tf.abs(c1-final_r1),-low_conf_mask+1)) 147 | temporal_g_loss = tf.reduce_mean(tf.abs(input_target[:,:,:,3:6]-final_r1)) + \ 148 | tf.reduce_mean(tf.multiply(tf.abs(warp_C0-final_r1),low_conf_mask)) + \ 149 | tf.reduce_mean(tf.multiply(tf.abs(c1-final_r1),-low_conf_mask+1)) 150 | 151 | opt=tf.train.AdamOptimizer(learning_rate=0.0001).minimize(lossDict["total"],var_list=[var for var in tf.trainable_variables()]) 152 | opt2=tf.train.AdamOptimizer(learning_rate=0.0001).minimize(0.2*lossDict["RankDiv_im1"],var_list=[var for var in tf.trainable_variables()]) 153 | opt_refine=tf.train.AdamOptimizer(learning_rate=0.0001).minimize(temporal_g_loss,var_list=[var for var in tf.trainable_variables() if var.name.startswith('VCRN')]) 154 | 155 | 156 | print([var for var in tf.trainable_variables() if var.name.startswith('VCRN')]) 157 | 158 | saver=tf.train.Saver(max_to_keep=1000) 159 | sess.run([tf.global_variables_initializer()]) 160 | 161 | var_restore = [v for v in tf.trainable_variables()] 162 | saver_restore=tf.train.Saver(var_restore) 163 | ckpt=tf.train.get_checkpoint_state(model) 164 | print("contain checkpoint: ", ckpt) 165 | if ckpt and continue_training: 166 | print('loaded '+ ckpt.model_checkpoint_path) 167 | saver_restore.restore(sess,ckpt.model_checkpoint_path) 168 | 169 | neigh=NearestNeighbors(n_neighbors=5) 170 | maxepoch=1001 171 | num_train=len(train_low) 172 | print("Number of training images: ", num_train) 173 | 174 | print("is_training: ", is_training ) 175 | if is_training: 176 | for epoch in range(1,maxepoch): 177 | print("Processing epoch %d"%epoch) 178 | input_list_src=[None]*num_train 179 | input_list_target=[None]*num_train 180 | input_list_flow_forward=[None]*num_train 181 | input_list_flow_backward=[None]*num_train 182 | gray_list_flow_forward=[None]*num_train 183 | gray_list_flow_backward=[None]*num_train 184 | if os.path.isdir("%s/%04d"%(model,epoch)): 185 | continue 186 | cnt=0 187 | all_RD,all_Bi = 0., 0. 188 | #Images 189 | for id in np.random.permutation(1431167):#1431167 190 | st=time.time() 191 | color_image=np.float32(scipy.misc.imread("%s/%06d.jpg"%(imgs_dir, id+1)))/255.0 192 | if len(color_image.shape)==2: 193 | continue 194 | h=color_image.shape[0]//32*32 195 | w=color_image.shape[1]//32*32 196 | color_image=color_image[:h:2,:w:2,:] 197 | gray_image=cv2.cvtColor(color_image,cv2.COLOR_RGB2GRAY) 198 | if gray_image is None: 199 | print(id) 200 | continue 201 | idxs = Bilateral_NN(color_image, neigh) 202 | gray_image=gray_image[:,:,np.newaxis] 203 | 204 | _, crt_RDLoss, crt_BiLoss = sess.run([opt2,lossDict["RankDiv_im1"], lossDict["Bilateral_im1"]],feed_dict={input_i:gray_image[np.newaxis,:,:,[0,0]],\ 205 | input_target:color_image[np.newaxis,:,:,[0,1,2,0,1,2]], input_idx:np.tile(idxs,[1,2])}) 206 | cnt+=1 207 | all_RD += crt_RDLoss 208 | all_Bi += crt_BiLoss 209 | print("Image iter: %d %d || RankDiv: %.4f %.4f|| Bi: %.4f %.4f || Time: %.4f"%(epoch,cnt,crt_RDLoss,all_RD/cnt,crt_BiLoss,all_Bi/cnt,time.time()-st)) 210 | if cnt>=5000: 211 | break 212 | 213 | # Video VCN 214 | cnt=0 215 | all_D1, all_D2, all_B1, all_B2, all_T, all_loss = 0,0,0,0,0,0 216 | for id in np.random.permutation(num_train): 217 | st=time.time() 218 | if input_list_src[id] is None: 219 | input_list_src[id], input_list_target[id], input_list_flow_forward[id], input_list_flow_backward[id] = prepare_input_w_flow(train_low[id], num_frames=num_frame) 220 | if input_list_src[id] is None or input_list_target[id] is None or input_list_flow_forward[id] is None: 221 | continue 222 | input_frames_processed = input_list_src[id] 223 | 224 | idxs1 = Bilateral_NN(input_list_target[id][0,:,:,:3], neigh) 225 | idxs2 = Bilateral_NN(input_list_target[id][0,:,:,3:6], neigh) 226 | 227 | _, out_loss, C0_im, C1_im = sess.run([opt, lossDict,C0,C1],\ 228 | feed_dict={input_i:input_list_src[id],input_target:input_list_target[id],\ 229 | input_flow_backward:input_list_flow_backward[id],\ 230 | input_idx: np.concatenate([idxs1,idxs2],axis=1)}) 231 | all_D1 += out_loss["RankDiv_im1"] 232 | all_D2 += out_loss["RankDiv_im2"] 233 | all_B1 += out_loss["Bilateral_im1"] 234 | all_B2 += out_loss["Bilateral_im2"] 235 | all_T += out_loss["temporal"] 236 | all_loss += out_loss["total"] 237 | 238 | cnt+=1 239 | print("iter: %d %d %.2fs loss: %.4f %.4f|| (D1) %.4f %.4f (D2) %.4f %.4f || (B1) %.4f %.4f (B2) %.4f %.4f (T) %.4f %.4f"\ 240 | %(epoch,cnt,out_loss["total"],all_loss/cnt, time.time()-st,\ 241 | out_loss["RankDiv_im1"], all_D1/cnt, out_loss["RankDiv_im2"], all_D2/cnt,\ 242 | out_loss["Bilateral_im1"], all_B1/cnt, out_loss["Bilateral_im2"], all_B2/cnt,\ 243 | out_loss["temporal"], all_T/cnt)) 244 | 245 | # Video Refine 246 | if epoch > 0: 247 | _, _, gray_list_flow_forward[id], gray_list_flow_backward[id] = prepare_input_w_flow(train_low[id], num_frames=num_frame) 248 | _, out_loss, final_C1 = sess.run([opt_refine, temporal_g_loss, final_r1],\ 249 | feed_dict={c0:C0_im[:,:,:,0:3], c1: C1_im[:,:,:,0:3], input_i:input_list_src[id],input_target:input_list_target[id],\ 250 | input_flow_backward:input_list_flow_backward[id],\ 251 | gray_flow_backward:gray_list_flow_backward[id],\ 252 | input_idx: np.concatenate([idxs1,idxs2],axis=1)}) 253 | print("iter: %d %d || Refine || loss: %.4f %.4f"%(epoch,cnt,out_loss,time.time()-st)) 254 | 255 | if cnt>=1000: 256 | break 257 | 258 | # Validation 259 | if not os.path.isdir("%s/%04d"%(model,epoch)): 260 | os.makedirs("%s/%04d"%(model,epoch)) 261 | 262 | if epoch % save_freq == 0: 263 | numtest=len(test_low) 264 | all_loss_test=np.zeros(numtest, dtype=float) 265 | for ind in range(numtest): 266 | if ind>30 and epoch%25>0: 267 | break 268 | input_image_src, input_image_target, input_flow_forward_src, input_flow_backward_src = prepare_input_w_flow(test_low[int(ind*60/pow(60,int(epoch%25==0)))],num_frames=num_frame) 269 | if input_image_src is None or input_image_target is None or input_flow_forward_src is None: 270 | print("Not able to read the images/flows.") 271 | flag=True 272 | continue 273 | st=time.time() 274 | C0_imall,C1_imall,C0_im, C1_im, warped,mask=sess.run([objDict["prediction_0"],objDict["prediction_1"],C0, C1,objDict["warped"],objDict['mask']],feed_dict={input_i:input_image_src, 275 | input_target:input_image_target, 276 | input_flow_backward:input_flow_backward_src 277 | }) 278 | print("test time for %s --> %.3f"%(ind, time.time()-st)) 279 | input_image_src, input_image_target, gray_flow_forward_src, gray_flow_backward_src = prepare_input_w_flow(test_low[int(ind*60/pow(60,int(epoch%25==0)))],num_frames=num_frame) 280 | h,w = C0_im.shape[1:3] 281 | outputs= [] 282 | for ref_i in range(4): 283 | output, out_cmap_C, out_cmap_X, out_low_conf_mask = sess.run([final_r1, cmap_C,cmap_X,low_conf_mask],feed_dict={c0:C0_im[:,:,:,ref_i*3:ref_i*3+3], c1:C1_im[:,:,:,ref_i*3:ref_i*3+3], \ 284 | input_i:input_image_src, input_target:input_image_target, \ 285 | gray_flow_backward:gray_flow_backward_src, input_flow_backward:input_flow_backward_src}) 286 | 287 | outputs.append(output[0,:,:,:]) 288 | # Debug 289 | 290 | if not os.path.isdir("%s/%04d/predictions" % (model, epoch)): 291 | os.makedirs("%s/%04d/predictions" % (model, epoch)) 292 | sic.imsave("%s/%04d/predictions/mask_%06d.jpg"%(model, epoch, ind),np.uint8(np.maximum(np.minimum(np.concatenate([out_cmap_C[0],out_cmap_X[0],out_low_conf_mask[0]],axis=1)* 255.0,255.0),0.0))) 293 | sic.imsave("%s/%04d/predictions/final_%06d.jpg"%(model, epoch, ind),np.uint8(np.maximum(np.minimum(output[0] * 255.0,255.0),0.0))) 294 | sic.imsave("%s/%04d/predictions/predictions_%06d.jpg"%(model, epoch, ind),np.uint8(np.maximum(np.minimum(C0_imall[0] * 255.0,255.0),0.0))) 295 | 296 | saver.save(sess,"%s/model.ckpt"%model) 297 | if epoch%10==0: 298 | saver.save(sess,"%s/%04d/model.ckpt"%(model,epoch)) 299 | 300 | # Inference 301 | else: 302 | test_low=utils.get_names(test_dir) 303 | numtest=len(test_low) 304 | print(test_low[0]) 305 | out_folder = test_dir.split('/')[-1] 306 | outputs= [None]*4 307 | for ind in range(numtest): 308 | input_image_src, input_image_target, input_flow_forward_src, input_flow_backward_src = prepare_input_w_flow(test_low[ind],num_frames=num_frame,gray=True) 309 | if input_image_src is None or input_image_target is None or input_flow_forward_src is None: 310 | print("Not able to read the images/flows.") 311 | continue 312 | st=time.time() 313 | C0_imall,C1_imall,C0_im, C1_im=sess.run([objDict["prediction_0"],objDict["prediction_1"],C0, C1],feed_dict={input_i:input_image_src, 314 | input_target:input_image_target, 315 | input_flow_backward:input_flow_backward_src 316 | }) 317 | print("test time for %s --> %.3f"%(ind, time.time()-st)) 318 | h,w = C0_im.shape[1:3] 319 | print(C0_im.shape) 320 | if not os.path.isdir("%s/%s" % (model, out_folder)): 321 | os.makedirs("%s/%s/predictions" % (model, out_folder)) 322 | os.makedirs("%s/%s/predictions0" % (model, out_folder)) 323 | os.makedirs("%s/%s/predictions1" % (model, out_folder)) 324 | os.makedirs("%s/%s/predictions2" % (model, out_folder)) 325 | os.makedirs("%s/%s/predictions3" % (model, out_folder)) 326 | if ind == 0: 327 | for ref_i in range(4): 328 | output,_ = sess.run([final_r1, temporal_g_loss],feed_dict={c0:C0_im[:,:,:,ref_i*3:ref_i*3+3], c1:C1_im[:,:,:,ref_i*3:ref_i*3+3], \ 329 | input_i:input_image_src, input_target:input_image_target, \ 330 | gray_flow_backward:input_flow_backward_src, input_flow_backward:input_flow_backward_src}) 331 | outputs[ref_i] = output 332 | sic.imsave("%s/%s/predictions%d/final_%06d.jpg"%(model, out_folder, ref_i, ind),np.uint8(np.maximum(np.minimum(C0_im[0,:,:,ref_i*3:ref_i*3+3] * 255.0,255.0),0.0))) 333 | sic.imsave("%s/%s/predictions%d/final_%06d.jpg"%(model, out_folder, ref_i, ind+1),np.uint8(np.maximum(np.minimum(output[0,:,:,:] * 255.0,255.0),0.0))) 334 | sic.imsave("%s/%s/predictions/predictions_%06d.jpg"%(model, out_folder, ind+1),np.uint8(np.maximum(np.minimum(C1_imall[0,:,:,:] * 255.0,255.0),0.0))) 335 | sic.imsave("%s/%s/predictions/final_%06d.jpg"%(model, out_folder, ind+1),np.uint8(np.maximum(np.minimum(np.concatenate(outputs,axis=2)[0,:,:,:] * 255.0,255.0),0.0))) 336 | 337 | else: 338 | for ref_i in range(4): 339 | output,_ = sess.run([final_r1, temporal_g_loss],feed_dict={c0:outputs[ref_i], c1:C1_im[:,:,:,:3], \ 340 | input_i:input_image_src, input_target:input_image_target, \ 341 | gray_flow_backward:input_flow_backward_src, input_flow_backward:input_flow_backward_src}) 342 | outputs.append(output[0,:,:,:]) 343 | sic.imsave("%s/%s/predictions%d/final_%06d.jpg"%(model, out_folder, ref_i, ind+1),np.uint8(np.maximum(np.minimum(output[0,:,:,:] * 255.0,255.0),0.0))) 344 | sic.imsave("%s/%s/predictions/predictions_%06d.jpg"%(model, out_folder, ind+1),np.uint8(np.maximum(np.minimum(C1_imall[0,:,:,:] * 255.0,255.0),0.0))) 345 | -------------------------------------------------------------------------------- /myflowlib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | # ============================== 4 | # flowlib.py 5 | # library for optical flow processing 6 | # Author: Ruoteng Li 7 | # Date: 6th Aug 2016 8 | # ============================== 9 | """ 10 | import png 11 | import numpy as np 12 | import matplotlib.colors as cl 13 | import matplotlib.pyplot as plt 14 | from PIL import Image 15 | 16 | 17 | UNKNOWN_FLOW_THRESH = 1e7 18 | SMALLFLOW = 0.0 19 | LARGEFLOW = 1e8 20 | 21 | """ 22 | ============= 23 | Flow Section 24 | ============= 25 | """ 26 | 27 | 28 | def show_flow(filename): 29 | """ 30 | visualize optical flow map using matplotlib 31 | :param filename: optical flow file 32 | :return: None 33 | """ 34 | flow = read_flow(filename) 35 | img = flow_to_image(flow) 36 | plt.imshow(img) 37 | plt.show() 38 | 39 | 40 | def visualize_flow(flow, mode='Y'): 41 | """ 42 | this function visualize the input flow 43 | :param flow: input flow in array 44 | :param mode: choose which color mode to visualize the flow (Y: Ccbcr, RGB: RGB color) 45 | :return: None 46 | """ 47 | if mode == 'Y': 48 | # Ccbcr color wheel 49 | img = flow_to_image(flow) 50 | plt.imshow(img) 51 | plt.show() 52 | elif mode == 'RGB': 53 | (h, w) = flow.shape[0:2] 54 | du = flow[:, :, 0] 55 | dv = flow[:, :, 1] 56 | valid = flow[:, :, 2] 57 | max_flow = max(np.max(du), np.max(dv)) 58 | img = np.zeros((h, w, 3), dtype=np.float64) 59 | # angle layer 60 | img[:, :, 0] = np.arctan2(dv, du) / (2 * np.pi) 61 | # magnitude layer, normalized to 1 62 | img[:, :, 1] = np.sqrt(du * du + dv * dv) * 8 / max_flow 63 | # phase layer 64 | img[:, :, 2] = 8 - img[:, :, 1] 65 | # clip to [0,1] 66 | small_idx = img[:, :, 0:3] < 0 67 | large_idx = img[:, :, 0:3] > 1 68 | img[small_idx] = 0 69 | img[large_idx] = 1 70 | # convert to rgb 71 | img = cl.hsv_to_rgb(img) 72 | # remove invalid point 73 | img[:, :, 0] = img[:, :, 0] * valid 74 | img[:, :, 1] = img[:, :, 1] * valid 75 | img[:, :, 2] = img[:, :, 2] * valid 76 | # show 77 | plt.imshow(img) 78 | plt.show() 79 | 80 | return None 81 | 82 | 83 | def read_flow(filename): 84 | """ 85 | read optical flow from Middlebury .flo file 86 | :param filename: name of the flow file 87 | :return: optical flow data in matrix 88 | """ 89 | f = open(filename, 'rb') 90 | magic = np.fromfile(f, np.float32, count=1) 91 | data2d = None 92 | 93 | if 202021.25 != magic: 94 | print ('Magic number incorrect. Invalid .flo file') 95 | else: 96 | w = np.fromfile(f, np.int32, count=1)[0] 97 | h = np.fromfile(f, np.int32, count=1)[0] 98 | # print ("Reading %d x %d flo file" % (h, w)) 99 | data2d = np.fromfile(f, np.float32, count=2 * w * h) 100 | # reshape data into 3D array (columns, rows, channels) 101 | data2d = np.resize(data2d, (h, w, 2)) 102 | f.close() 103 | return data2d 104 | 105 | 106 | def read_flow_png(flow_file): 107 | """ 108 | Read optical flow from KITTI .png file 109 | :param flow_file: name of the flow file 110 | :return: optical flow data in matrix 111 | """ 112 | flow_object = png.Reader(filename=flow_file) 113 | flow_direct = flow_object.asDirect() 114 | flow_data = list(flow_direct[2]) 115 | (w, h) = flow_direct[3]['size'] 116 | flow = np.zeros((h, w, 3), dtype=np.float64) 117 | for i in range(len(flow_data)): 118 | flow[i, :, 0] = flow_data[i][0::3] 119 | flow[i, :, 1] = flow_data[i][1::3] 120 | flow[i, :, 2] = flow_data[i][2::3] 121 | 122 | invalid_idx = (flow[:, :, 2] == 0) 123 | flow[:, :, 0:2] = (flow[:, :, 0:2] - 2 ** 15) / 64.0 124 | flow[invalid_idx, 0] = 0 125 | flow[invalid_idx, 1] = 0 126 | return flow 127 | 128 | def write_flow_round(flow, filename): 129 | """ 130 | write optical flow in Middlebury .flo format 131 | :param flow: optical flow map 132 | :param filename: optical flow file path to be saved 133 | :return: None 134 | """ 135 | f = open(filename, 'wb') 136 | flow = (flow*100).astype(np.int16) 137 | magic = np.array([202021.25], dtype=np.float32) 138 | (height, width) = flow.shape[0:2] 139 | w = np.array([width], dtype=np.int32) 140 | h = np.array([height], dtype=np.int32) 141 | magic.tofile(f) 142 | w.tofile(f) 143 | h.tofile(f) 144 | flow.tofile(f) 145 | f.close() 146 | 147 | def read_flow_round(filename): 148 | """ 149 | read optical flow from Middlebury .flo file 150 | :param filename: name of the flow file 151 | :return: optical flow data in matrix 152 | """ 153 | f = open(filename, 'rb') 154 | magic = np.fromfile(f, np.float32, count=1) 155 | data2d = None 156 | 157 | if 202021.25 != magic: 158 | print ('Magic number incorrect. Invalid .flo file') 159 | else: 160 | w = np.fromfile(f, np.int32, count=1)[0] 161 | h = np.fromfile(f, np.int32, count=1)[0] 162 | # print ("Reading %d x %d flo file" % (h, w)) 163 | data2d = np.fromfile(f, np.int16, count=2 * w * h) 164 | # reshape data into 3D array (columns, rows, channels) 165 | data2d = np.resize(data2d, (h, w, 2)) 166 | data2d = data2d.astype(np.float32)/100 167 | f.close() 168 | return data2d 169 | 170 | def write_flow(flow, filename): 171 | """ 172 | write optical flow in Middlebury .flo format 173 | :param flow: optical flow map 174 | :param filename: optical flow file path to be saved 175 | :return: None 176 | """ 177 | f = open(filename, 'wb') 178 | magic = np.array([202021.25], dtype=np.float32) 179 | (height, width) = flow.shape[0:2] 180 | w = np.array([width], dtype=np.int32) 181 | h = np.array([height], dtype=np.int32) 182 | magic.tofile(f) 183 | w.tofile(f) 184 | h.tofile(f) 185 | flow.tofile(f) 186 | f.close() 187 | 188 | def segment_flow(flow): 189 | h = flow.shape[0] 190 | w = flow.shape[1] 191 | u = flow[:, :, 0] 192 | v = flow[:, :, 1] 193 | 194 | idx = ((abs(u) > LARGEFLOW) | (abs(v) > LARGEFLOW)) 195 | idx2 = (abs(u) == SMALLFLOW) 196 | class0 = (v == 0) & (u == 0) 197 | u[idx2] = 0.00001 198 | tan_value = v / u 199 | 200 | class1 = (tan_value < 1) & (tan_value >= 0) & (u > 0) & (v >= 0) 201 | class2 = (tan_value >= 1) & (u >= 0) & (v >= 0) 202 | class3 = (tan_value < -1) & (u <= 0) & (v >= 0) 203 | class4 = (tan_value < 0) & (tan_value >= -1) & (u < 0) & (v >= 0) 204 | class8 = (tan_value >= -1) & (tan_value < 0) & (u > 0) & (v <= 0) 205 | class7 = (tan_value < -1) & (u >= 0) & (v <= 0) 206 | class6 = (tan_value >= 1) & (u <= 0) & (v <= 0) 207 | class5 = (tan_value >= 0) & (tan_value < 1) & (u < 0) & (v <= 0) 208 | 209 | seg = np.zeros((h, w)) 210 | 211 | seg[class1] = 1 212 | seg[class2] = 2 213 | seg[class3] = 3 214 | seg[class4] = 4 215 | seg[class5] = 5 216 | seg[class6] = 6 217 | seg[class7] = 7 218 | seg[class8] = 8 219 | seg[class0] = 0 220 | seg[idx] = 0 221 | 222 | return seg 223 | 224 | 225 | def flow_error(tu, tv, u, v): 226 | """ 227 | Calculate average end point error 228 | :param tu: ground-truth horizontal flow map 229 | :param tv: ground-truth vertical flow map 230 | :param u: estimated horizontal flow map 231 | :param v: estimated vertical flow map 232 | :return: End point error of the estimated flow 233 | """ 234 | smallflow = 0.0 235 | ''' 236 | stu = tu[bord+1:end-bord,bord+1:end-bord] 237 | stv = tv[bord+1:end-bord,bord+1:end-bord] 238 | su = u[bord+1:end-bord,bord+1:end-bord] 239 | sv = v[bord+1:end-bord,bord+1:end-bord] 240 | ''' 241 | stu = tu[:] 242 | stv = tv[:] 243 | su = u[:] 244 | sv = v[:] 245 | 246 | idxUnknow = (abs(stu) > UNKNOWN_FLOW_THRESH) | (abs(stv) > UNKNOWN_FLOW_THRESH) 247 | stu[idxUnknow] = 0 248 | stv[idxUnknow] = 0 249 | su[idxUnknow] = 0 250 | sv[idxUnknow] = 0 251 | 252 | ind2 = [(np.absolute(stu) > smallflow) | (np.absolute(stv) > smallflow)] 253 | index_su = su[ind2] 254 | index_sv = sv[ind2] 255 | an = 1.0 / np.sqrt(index_su ** 2 + index_sv ** 2 + 1) 256 | un = index_su * an 257 | vn = index_sv * an 258 | 259 | index_stu = stu[ind2] 260 | index_stv = stv[ind2] 261 | tn = 1.0 / np.sqrt(index_stu ** 2 + index_stv ** 2 + 1) 262 | tun = index_stu * tn 263 | tvn = index_stv * tn 264 | 265 | ''' 266 | angle = un * tun + vn * tvn + (an * tn) 267 | index = [angle == 1.0] 268 | angle[index] = 0.999 269 | ang = np.arccos(angle) 270 | mang = np.mean(ang) 271 | mang = mang * 180 / np.pi 272 | ''' 273 | 274 | epe = np.sqrt((stu - su) ** 2 + (stv - sv) ** 2) 275 | epe = epe[ind2] 276 | mepe = np.mean(epe) 277 | return mepe 278 | 279 | 280 | def flow_to_image(flow): 281 | """ 282 | Convert flow into middlebury color code image 283 | :param flow: optical flow map 284 | :return: optical flow image in middlebury color 285 | """ 286 | u = flow[:, :, 0] 287 | v = flow[:, :, 1] 288 | 289 | maxu = -999. 290 | maxv = -999. 291 | minu = 999. 292 | minv = 999. 293 | 294 | idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH) 295 | u[idxUnknow] = 0 296 | v[idxUnknow] = 0 297 | 298 | maxu = max(maxu, np.max(u)) 299 | minu = min(minu, np.min(u)) 300 | 301 | maxv = max(maxv, np.max(v)) 302 | minv = min(minv, np.min(v)) 303 | 304 | rad = np.sqrt(u ** 2 + v ** 2) 305 | maxrad = max(-1, np.max(rad)) 306 | 307 | print ("max flow: %.4f\nflow range:\nu = %.3f .. %.3f\nv = %.3f .. %.3f" % (maxrad, minu,maxu, minv, maxv)) 308 | 309 | u = u/(maxrad + np.finfo(float).eps) 310 | v = v/(maxrad + np.finfo(float).eps) 311 | 312 | img = compute_color(u, v) 313 | 314 | idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2) 315 | img[idx] = 0 316 | 317 | return np.uint8(img) 318 | 319 | 320 | def evaluate_flow_file(gt, pred): 321 | """ 322 | evaluate the estimated optical flow end point error according to ground truth provided 323 | :param gt: ground truth file path 324 | :param pred: estimated optical flow file path 325 | :return: end point error, float32 326 | """ 327 | # Read flow files and calculate the errors 328 | gt_flow = read_flow(gt) # ground truth flow 329 | eva_flow = read_flow(pred) # predicted flow 330 | # Calculate errors 331 | average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], eva_flow[:, :, 0], eva_flow[:, :, 1]) 332 | return average_pe 333 | 334 | 335 | def evaluate_flow(gt_flow, pred_flow): 336 | """ 337 | gt: ground-truth flow 338 | pred: estimated flow 339 | """ 340 | average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], pred_flow[:, :, 0], pred_flow[:, :, 1]) 341 | return average_pe 342 | 343 | 344 | """ 345 | ============== 346 | Disparity Section 347 | ============== 348 | """ 349 | 350 | 351 | def read_disp_png(file_name): 352 | """ 353 | Read optical flow from KITTI .png file 354 | :param file_name: name of the flow file 355 | :return: optical flow data in matrix 356 | """ 357 | image_object = png.Reader(filename=file_name) 358 | image_direct = image_object.asDirect() 359 | image_data = list(image_direct[2]) 360 | (w, h) = image_direct[3]['size'] 361 | channel = len(image_data[0]) / w 362 | flow = np.zeros((h, w, channel), dtype=np.uint16) 363 | for i in range(len(image_data)): 364 | for j in range(channel): 365 | flow[i, :, j] = image_data[i][j::channel] 366 | return flow[:, :, 0] / 256 367 | 368 | 369 | def disp_to_flowfile(disp, filename): 370 | """ 371 | Read KITTI disparity file in png format 372 | :param disp: disparity matrix 373 | :param filename: the flow file name to save 374 | :return: None 375 | """ 376 | f = open(filename, 'wb') 377 | magic = np.array([202021.25], dtype=np.float32) 378 | (height, width) = disp.shape[0:2] 379 | w = np.array([width], dtype=np.int32) 380 | h = np.array([height], dtype=np.int32) 381 | empty_map = np.zeros((height, width), dtype=np.float32) 382 | data = np.dstack((disp, empty_map)) 383 | magic.tofile(f) 384 | w.tofile(f) 385 | h.tofile(f) 386 | data.tofile(f) 387 | f.close() 388 | 389 | 390 | """ 391 | ============== 392 | Image Section 393 | ============== 394 | """ 395 | 396 | 397 | def read_image(filename): 398 | """ 399 | Read normal image of any format 400 | :param filename: name of the image file 401 | :return: image data in matrix uint8 type 402 | """ 403 | img = Image.open(filename) 404 | im = np.array(img) 405 | return im 406 | 407 | 408 | def warp_image(im, flow): 409 | """ 410 | Use optical flow to warp image to the next 411 | :param im: image to warp 412 | :param flow: optical flow 413 | :return: warped image 414 | """ 415 | from scipy import interpolate 416 | image_height = im.shape[0] 417 | image_width = im.shape[1] 418 | flow_height = flow.shape[0] 419 | flow_width = flow.shape[1] 420 | n = image_height * image_width 421 | (iy, ix) = np.mgrid[0:image_height, 0:image_width] 422 | (fy, fx) = np.mgrid[0:flow_height, 0:flow_width] 423 | # print(flow.shape, flow.shape) 424 | fx = fx + flow[:,:,0] 425 | fy = fy + flow[:,:,1] 426 | mask = np.logical_or(fx <0 , fx > flow_width) 427 | mask = np.logical_or(mask, fy < 0) 428 | mask = np.logical_or(mask, fy > flow_height) 429 | fx = np.minimum(np.maximum(fx, 0), flow_width) 430 | fy = np.minimum(np.maximum(fy, 0), flow_height) 431 | points = np.concatenate((ix.reshape(n,1), iy.reshape(n,1)), axis=1) 432 | xi = np.concatenate((fx.reshape(n, 1), fy.reshape(n,1)), axis=1) 433 | warp = np.zeros((image_height, image_width, im.shape[2])) 434 | for i in range(im.shape[2]): 435 | channel = im[:, :, i] 436 | plt.imshow(channel, cmap='gray') 437 | values = channel.reshape(n, 1) 438 | new_channel = interpolate.griddata(points, values, xi, method='cubic') 439 | new_channel = np.reshape(new_channel, [flow_height, flow_width]) 440 | new_channel[mask] = 1 441 | warp[:, :, i] = new_channel.astype(np.uint8) 442 | 443 | return warp.astype(np.uint8) 444 | 445 | 446 | """ 447 | ============== 448 | Others 449 | ============== 450 | """ 451 | 452 | def scale_image(image, new_range): 453 | """ 454 | Linearly scale the image into desired range 455 | :param image: input image 456 | :param new_range: the new range to be aligned 457 | :return: image normalized in new range 458 | """ 459 | min_val = np.min(image).astype(np.float32) 460 | max_val = np.max(image).astype(np.float32) 461 | min_val_new = np.array(min(new_range), dtype=np.float32) 462 | max_val_new = np.array(max(new_range), dtype=np.float32) 463 | scaled_image = (image - min_val) / (max_val - min_val) * (max_val_new - min_val_new) + min_val_new 464 | return scaled_image.astype(np.uint8) 465 | 466 | 467 | def compute_color(u, v): 468 | """ 469 | compute optical flow color map 470 | :param u: optical flow horizontal map 471 | :param v: optical flow vertical map 472 | :return: optical flow in color code 473 | """ 474 | [h, w] = u.shape 475 | img = np.zeros([h, w, 3]) 476 | nanIdx = np.isnan(u) | np.isnan(v) 477 | u[nanIdx] = 0 478 | v[nanIdx] = 0 479 | 480 | colorwheel = make_color_wheel() 481 | ncols = np.size(colorwheel, 0) 482 | 483 | rad = np.sqrt(u**2+v**2) 484 | 485 | a = np.arctan2(-v, -u) / np.pi 486 | 487 | fk = (a+1) / 2 * (ncols - 1) + 1 488 | 489 | k0 = np.floor(fk).astype(int) 490 | 491 | k1 = k0 + 1 492 | k1[k1 == ncols+1] = 1 493 | f = fk - k0 494 | 495 | for i in range(0, np.size(colorwheel,1)): 496 | tmp = colorwheel[:, i] 497 | col0 = tmp[k0-1] / 255 498 | col1 = tmp[k1-1] / 255 499 | col = (1-f) * col0 + f * col1 500 | 501 | idx = rad <= 1 502 | col[idx] = 1-rad[idx]*(1-col[idx]) 503 | notidx = np.logical_not(idx) 504 | 505 | col[notidx] *= 0.75 506 | img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx))) 507 | 508 | return img 509 | 510 | 511 | def make_color_wheel(): 512 | """ 513 | Generate color wheel according Middlebury color code 514 | :return: Color wheel 515 | """ 516 | RY = 15 517 | YG = 6 518 | GC = 4 519 | CB = 11 520 | BM = 13 521 | MR = 6 522 | 523 | ncols = RY + YG + GC + CB + BM + MR 524 | 525 | colorwheel = np.zeros([ncols, 3]) 526 | 527 | col = 0 528 | 529 | # RY 530 | colorwheel[0:RY, 0] = 255 531 | colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY)) 532 | col += RY 533 | 534 | # YG 535 | colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG)) 536 | colorwheel[col:col+YG, 1] = 255 537 | col += YG 538 | 539 | # GC 540 | colorwheel[col:col+GC, 1] = 255 541 | colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC)) 542 | col += GC 543 | 544 | # CB 545 | colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB)) 546 | colorwheel[col:col+CB, 2] = 255 547 | col += CB 548 | 549 | # BM 550 | colorwheel[col:col+BM, 2] = 255 551 | colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM)) 552 | col += + BM 553 | 554 | # MR 555 | colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR)) 556 | colorwheel[col:col+MR, 0] = 255 557 | 558 | return colorwheel 559 | -------------------------------------------------------------------------------- /network.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import os,time,cv2 5 | from scipy import io 6 | import tensorflow as tf 7 | import tensorflow.contrib.slim as slim 8 | import numpy as np 9 | import subprocess 10 | import utils as utils 11 | 12 | 13 | def identity_initializer(): 14 | def _initializer(shape, dtype=tf.float32, partition_info=None): 15 | array = np.zeros(shape, dtype=float) 16 | cx, cy = shape[0]//2, shape[1]//2 17 | for i in range(np.minimum(shape[2],shape[3])): 18 | array[cx, cy, i, i] = 1 19 | return tf.constant(array, dtype=dtype) 20 | return _initializer 21 | 22 | def lrelu(x): 23 | return tf.maximum(x*0.2,x) 24 | 25 | def bilinear_up_and_concat(x1, x2, output_channels, in_channels, scope): 26 | with tf.variable_scope(scope): 27 | upconv = tf.image.resize_images(x1, [tf.shape(x1)[1]*2, tf.shape(x1)[2]*2] ) 28 | upconv.set_shape([None, None, None, in_channels]) 29 | upconv = slim.conv2d(upconv,output_channels,[3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope='up_conv1') 30 | upconv_output = tf.concat([upconv, x2], axis=3) 31 | upconv_output.set_shape([None, None, None, output_channels*2]) 32 | return upconv_output 33 | 34 | def bottleneck_block(input, channel=32, ext='g_conv'): 35 | conv1_1=slim.conv2d(input, channel // 4, [1,1], rate=1, activation_fn=lrelu, scope=ext +'_1_1') 36 | conv1_2=slim.conv2d(conv1_1, channel // 4, [3,3], rate=1, activation_fn=lrelu, scope=ext +'_1_2') 37 | conv1_3=slim.conv2d(conv1_2, channel, [1,1], rate=1, activation_fn=lrelu, scope=ext +'_1_3') 38 | return conv1_3 39 | 40 | def bottleneck_unet(input, channel=32, output_channel=3,reuse=False,ext="",div_num=1): 41 | if reuse: 42 | tf.get_variable_scope().reuse_variables() 43 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, scope=ext+'g_conv1_1') 44 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, scope=ext+'g_conv1_2') 45 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 46 | 47 | conv2 = bottleneck_block(pool1, channel * 2, ext='g_conv2') 48 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 49 | 50 | conv3 = bottleneck_block(pool2, channel * 4, ext='g_conv3') 51 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 52 | 53 | conv4 = bottleneck_block(pool3, channel * 8, ext='g_conv4') 54 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 55 | 56 | conv5 = bottleneck_block(pool4, channel * 16, ext='g_conv5') 57 | 58 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"g_up_1" ) 59 | conv6 = bottleneck_block(up6, channel * 8, ext='g_conv6') 60 | 61 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"g_up_2" ) 62 | conv7 = bottleneck_block(up7, channel * 4, ext='g_conv7') 63 | 64 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"g_up_3" ) 65 | conv8 = bottleneck_block(up8, channel * 2, ext='g_conv8') 66 | 67 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"g_up_4" ) 68 | 69 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_1') 70 | conv9=slim.conv2d(conv9,output_channel*div_num,[3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_2') 71 | return conv9 72 | 73 | 74 | def dowmsample_unet(input, channel=32, output_channel=3, reuse=False, ext="",div_num=1): 75 | """ 76 | docstring 77 | """ 78 | if reuse: 79 | tf.get_variable_scope().reuse_variables() 80 | 81 | down_input = tf.image.resize_images(input, [tf.shape(input)[1] // 2, tf.shape(input)[2] // 2]) 82 | down_hyper = utils.build(tf.tile(down_input, [1,1,1,3])) 83 | 84 | conv0=slim.conv2d(input, 32, [3,3], rate=1, activation_fn=lrelu, scope=ext+'g_conv0_1') 85 | conv0=slim.conv2d(conv0, 32, [3,3], rate=1, activation_fn=lrelu, scope=ext+'g_conv0_2') 86 | pool0=slim.max_pool2d(conv0, [2, 2], padding='SAME' ) 87 | 88 | net_input = tf.concat([pool0, down_hyper], axis=3) 89 | net_output = VCN(net_input, output_channel = 64, reuse=reuse, div_num=div_num) 90 | 91 | 92 | up10 = bilinear_up_and_concat(net_output, conv0, channel, channel*2, scope=ext+"g_up_5" ) 93 | conv10=slim.conv2d(up10, channel, [3,3], rate=1, activation_fn=lrelu, scope=ext+'g_conv10_1') 94 | conv10=slim.conv2d(conv10, output_channel * div_num, [3,3], rate=1, activation_fn=None, scope=ext+'g_conv10_2') 95 | return conv10 96 | 97 | def plain_unet(input, channel=32, output_channel=3, reuse=False,ext="", div_num=1): 98 | if reuse: 99 | tf.get_variable_scope().reuse_variables() 100 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_1') 101 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_2') 102 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 103 | conv2=slim.conv2d(pool1,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_1') 104 | conv2=slim.conv2d(conv2,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_2') 105 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 106 | conv3=slim.conv2d(pool2,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_1') 107 | conv3=slim.conv2d(conv3,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_2') 108 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 109 | conv4=slim.conv2d(pool3,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_1') 110 | conv4=slim.conv2d(conv4,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_2') 111 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 112 | conv5=slim.conv2d(pool4,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_1') 113 | conv5=slim.conv2d(conv5,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_2') 114 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"g_up_1" ) 115 | conv6=slim.conv2d(up6, channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_1') 116 | conv6=slim.conv2d(conv6,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_2') 117 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"g_up_2" ) 118 | conv7=slim.conv2d(up7, channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_1') 119 | conv7=slim.conv2d(conv7,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_2') 120 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"g_up_3" ) 121 | conv8=slim.conv2d(up8, channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_1') 122 | conv8=slim.conv2d(conv8,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_2') 123 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"g_up_4" ) 124 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_1') 125 | conv9=slim.conv2d(conv9, output_channel*div_num, [3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_2') 126 | return conv9 127 | 128 | def hyper_unet(input, channel=32, output_channel=3, reuse=False,ext="", div_num=1): 129 | if reuse: 130 | tf.get_variable_scope().reuse_variables() 131 | input = utils.build(tf.tile(input, [1,1,1,3])) 132 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_1') 133 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_2') 134 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 135 | conv2=slim.conv2d(pool1,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_1') 136 | conv2=slim.conv2d(conv2,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_2') 137 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 138 | conv3=slim.conv2d(pool2,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_1') 139 | conv3=slim.conv2d(conv3,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_2') 140 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 141 | conv4=slim.conv2d(pool3,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_1') 142 | conv4=slim.conv2d(conv4,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_2') 143 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 144 | conv5=slim.conv2d(pool4,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_1') 145 | conv5=slim.conv2d(conv5,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_2') 146 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"g_up_1" ) 147 | conv6=slim.conv2d(up6, channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_1') 148 | conv6=slim.conv2d(conv6,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_2') 149 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"g_up_2" ) 150 | conv7=slim.conv2d(up7, channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_1') 151 | conv7=slim.conv2d(conv7,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_2') 152 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"g_up_3" ) 153 | conv8=slim.conv2d(up8, channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_1') 154 | conv8=slim.conv2d(conv8,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_2') 155 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"g_up_4" ) 156 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_1') 157 | conv9=slim.conv2d(conv9, output_channel*div_num, [3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_2') 158 | return conv9 159 | 160 | 161 | def VCN(input, channel=32, output_channel=3, reuse=False,ext="", div_num=1): 162 | if reuse: 163 | tf.get_variable_scope().reuse_variables() 164 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_1') 165 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_2') 166 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 167 | conv2=slim.conv2d(pool1,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_1') 168 | conv2=slim.conv2d(conv2,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_2') 169 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 170 | conv3=slim.conv2d(pool2,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_1') 171 | conv3=slim.conv2d(conv3,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_2') 172 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 173 | conv4=slim.conv2d(pool3,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_1') 174 | conv4=slim.conv2d(conv4,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_2') 175 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 176 | conv5=slim.conv2d(pool4,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_1') 177 | conv5=slim.conv2d(conv5,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_2') 178 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"g_up_1" ) 179 | conv6=slim.conv2d(up6, channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_1') 180 | conv6=slim.conv2d(conv6,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_2') 181 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"g_up_2" ) 182 | conv7=slim.conv2d(up7, channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_1') 183 | conv7=slim.conv2d(conv7,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_2') 184 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"g_up_3" ) 185 | conv8=slim.conv2d(up8, channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_1') 186 | conv8=slim.conv2d(conv8,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_2') 187 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"g_up_4" ) 188 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_1') 189 | conv9=slim.conv2d(conv9, output_channel*div_num, [3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_2') 190 | return conv9 191 | 192 | def VCRN(input, channel=32, output_channel=3,reuse=False,ext="VCRN"): 193 | if reuse: 194 | tf.get_variable_scope().reuse_variables() 195 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv1_1') 196 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv1_2') 197 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 198 | conv2=slim.conv2d(pool1,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv2_1') 199 | conv2=slim.conv2d(conv2,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv2_2') 200 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 201 | conv3=slim.conv2d(pool2,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv3_1') 202 | conv3=slim.conv2d(conv3,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv3_2') 203 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 204 | conv4=slim.conv2d(pool3,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv4_1') 205 | conv4=slim.conv2d(conv4,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv4_2') 206 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 207 | conv5=slim.conv2d(pool4,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv5_1') 208 | conv5=slim.conv2d(conv5,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv5_2') 209 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"r_up_1" ) 210 | conv6=slim.conv2d(up6, channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv6_1') 211 | conv6=slim.conv2d(conv6,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv6_2') 212 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"r_up_2" ) 213 | conv7=slim.conv2d(up7, channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv7_1') 214 | conv7=slim.conv2d(conv7,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv7_2') 215 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"r_up_3" ) 216 | conv8=slim.conv2d(up8, channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv8_1') 217 | conv8=slim.conv2d(conv8,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv8_2') 218 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"r_up_4" ) 219 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv9_1') 220 | conv9=slim.conv2d(conv9,output_channel,[3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'r_conv9_2') 221 | return conv9 222 | 223 | 224 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | #tensorflow 1.2.0 is needed 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | import os 6 | import time 7 | import tensorflow as tf 8 | import tensorflow.contrib.slim as slim 9 | import numpy as np 10 | import utils as utils 11 | import subprocess 12 | import network as net 13 | import loss as loss 14 | import argparse 15 | from glob import glob 16 | import imageio 17 | import json 18 | import math 19 | import random 20 | from PIL import Image 21 | 22 | seed = 2020 23 | np.random.seed(seed) 24 | tf.set_random_seed(seed) 25 | random.seed(seed) 26 | 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument("--model", default='model_imagecolor', type=str, help="Model Name") 29 | parser.add_argument("--img_path", default='demo_imgs/ILSVRC2012_val_00040251.JPEG', type=str, help="Test dir path") 30 | parser.add_argument("--video_path", default=None, type=str, help="Test dir path") 31 | parser.add_argument('--network', type=str, default='half_hyper_unet', 32 | help='chooses which model to use. unet, fcn', 33 | choices=["half_hyper_unet", "hyper_unet"]) 34 | 35 | 36 | ARGS = parser.parse_args() 37 | print(ARGS) 38 | os.makedirs("./test_result/{}".format(ARGS.model), exist_ok=True) 39 | 40 | 41 | os.environ["CUDA_VISIBLE_DEVICES"]=str(np.argmax( [int(x.split()[2]) for x in subprocess.Popen("nvidia-smi -q -d Memory | grep -A4 GPU | grep Free", shell=True, stdout=subprocess.PIPE).stdout.readlines()])) 42 | config=tf.ConfigProto() 43 | config.gpu_options.allow_growth=True 44 | sess=tf.Session(config=config) 45 | 46 | 47 | gray1=tf.placeholder(tf.float32,shape=[None,None,None, 1]) 48 | rgb1=tf.placeholder(tf.float32,shape=[None,None,None, 3]) 49 | 50 | 51 | def get_model(gray1): 52 | if ARGS.network == "half_hyper_unet": 53 | pred1 = net.dowmsample_unet(gray1, reuse=False) 54 | elif ARGS.network == "hyper_unet": 55 | pred1 = net.VCN(utils.build(tf.tile(gray1, [1,1,1,3])), reuse=False, div_num=1) 56 | return pred1 57 | 58 | with tf.variable_scope(tf.get_variable_scope()): 59 | with tf.variable_scope('siamese_nework'): 60 | pred1 = get_model(gray1) 61 | 62 | saver=tf.train.Saver(max_to_keep=1000) 63 | sess.run([tf.global_variables_initializer()]) 64 | var_restore = [v for v in tf.trainable_variables()] 65 | saver_restore=tf.train.Saver(var_restore) 66 | ckpt=tf.train.get_checkpoint_state('./pretrained_models/' + ARGS.model) 67 | print("contain checkpoint: ", ckpt) 68 | if ckpt: 69 | print('loaded '+ ckpt.model_checkpoint_path) 70 | saver_restore.restore(sess, ckpt.model_checkpoint_path) 71 | else: 72 | print("there is no checkpoint: {}".format('./pretrained_models/' + ARGS.model)) 73 | 74 | def test(): 75 | if ARGS.video_path is None: 76 | val_names = [ARGS.img_path] 77 | else: 78 | val_names = sorted(glob(ARGS.video_path + '/*')) 79 | cnt = 0 80 | print(len(val_names)) 81 | for id in range(len(val_names)): 82 | gray_image=np.array(Image.open(val_names[id]).convert("L")) / 255. 83 | basename = val_names[id].split("/")[-1] 84 | h=gray_image.shape[0] // 32 * 32 85 | w=gray_image.shape[1] // 32 * 32 86 | pred_image = sess.run(pred1, feed_dict={gray1:gray_image[np.newaxis,:h,:w,np.newaxis]}) 87 | if ARGS.video_path is None: 88 | imageio.imwrite("./test_result/{}/{}".format(ARGS.model, basename.replace(".JPEG", "_gray.png")), gray_image[:h,:w]) 89 | imageio.imwrite("./test_result/{}/{}".format(ARGS.model, basename.replace(".JPEG", "_colorized.png")), pred_image[0]) 90 | else: 91 | base_path = ARGS.video_path[:-1] if ARGS.video_path.endswith("/") else ARGS.video_path 92 | os.makedirs(base_path + "_colorized", exist_ok=True) 93 | imageio.imwrite("{}_colorized/{}".format(base_path, basename), gray_image[:h,:w]) 94 | imageio.imwrite("{}_colorized/{}".format(base_path, basename), pred_image[0]) 95 | 96 | test() -------------------------------------------------------------------------------- /test_div_video.py: -------------------------------------------------------------------------------- 1 | #tensorflow 1.2.0 is needed 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | import os,time,scipy.io 6 | import tensorflow as tf 7 | import tensorflow.contrib.slim as slim 8 | import numpy as np 9 | import utils as utils 10 | import imageio 11 | import subprocess 12 | import argparse 13 | from PIL import Image 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--model", default='model_div_woflow', type=str, help="Model Name") 17 | parser.add_argument("--use_gpu", default=1, type=int, help="Use gpu or not") 18 | parser.add_argument("--video_path", default='demo_vid', type=str, help="Test video dir") 19 | parser.add_argument("--img_path", default=None, type=str, help="Test image path") 20 | parser.add_argument("--output_dir", default=None, type=str, help="Output frames dir") 21 | 22 | 23 | ARGS = parser.parse_args() 24 | video_path = ARGS.video_path 25 | img_path = ARGS.img_path 26 | model=ARGS.model 27 | print(ARGS) 28 | 29 | if not ARGS.use_gpu: 30 | os.environ["CUDA_VISIBLE_DEVICES"]='' 31 | else: 32 | os.environ["CUDA_VISIBLE_DEVICES"]=str(np.argmax( [int(x.split()[2]) for x in subprocess.Popen("nvidia-smi -q -d Memory | grep -A4 GPU | grep Free", shell=True, stdout=subprocess.PIPE).stdout.readlines()])) 33 | 34 | def identity_initializer(): 35 | def _initializer(shape, dtype=tf.float32, partition_info=None): 36 | array = np.zeros(shape, dtype=float) 37 | cx, cy = shape[0]//2, shape[1]//2 38 | for i in range(np.minimum(shape[2],shape[3])): 39 | array[cx, cy, i, i] = 1 40 | return tf.constant(array, dtype=dtype) 41 | return _initializer 42 | 43 | def lrelu(x): 44 | return tf.maximum(x*0.2,x) 45 | 46 | def bilinear_up_and_concat(x1, x2, output_channels, in_channels, scope): 47 | with tf.variable_scope(scope): 48 | upconv = tf.image.resize_images(x1, [tf.shape(x1)[1]*2, tf.shape(x1)[2]*2] ) 49 | upconv.set_shape([None, None, None, in_channels]) 50 | upconv = slim.conv2d(upconv,output_channels,[3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope='up_conv1') 51 | upconv_output = tf.concat([upconv, x2], axis=3) 52 | upconv_output.set_shape([None, None, None, output_channels*2]) 53 | return upconv_output 54 | 55 | def VCN(input, channel=32, output_channel=3,reuse=False,ext=""): 56 | if reuse: 57 | tf.get_variable_scope().reuse_variables() 58 | conv1=slim.conv2d(input,channel,[1,1], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_1') 59 | conv1=slim.conv2d(conv1,channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv1_2') 60 | pool1=slim.max_pool2d(conv1, [2, 2], padding='SAME' ) 61 | conv2=slim.conv2d(pool1,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_1') 62 | conv2=slim.conv2d(conv2,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv2_2') 63 | pool2=slim.max_pool2d(conv2, [2, 2], padding='SAME' ) 64 | conv3=slim.conv2d(pool2,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_1') 65 | conv3=slim.conv2d(conv3,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv3_2') 66 | pool3=slim.max_pool2d(conv3, [2, 2], padding='SAME' ) 67 | conv4=slim.conv2d(pool3,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_1') 68 | conv4=slim.conv2d(conv4,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv4_2') 69 | pool4=slim.max_pool2d(conv4, [2, 2], padding='SAME' ) 70 | conv5=slim.conv2d(pool4,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_1') 71 | conv5=slim.conv2d(conv5,channel*16,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv5_2') 72 | up6 = bilinear_up_and_concat( conv5, conv4, channel*8, channel*16, scope=ext+"g_up_1" ) 73 | conv6=slim.conv2d(up6, channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_1') 74 | conv6=slim.conv2d(conv6,channel*8,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv6_2') 75 | up7 = bilinear_up_and_concat( conv6, conv3, channel*4, channel*8, scope=ext+"g_up_2" ) 76 | conv7=slim.conv2d(up7, channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_1') 77 | conv7=slim.conv2d(conv7,channel*4,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv7_2') 78 | up8 = bilinear_up_and_concat( conv7, conv2, channel*2, channel*4, scope=ext+"g_up_3" ) 79 | conv8=slim.conv2d(up8, channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_1') 80 | conv8=slim.conv2d(conv8,channel*2,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv8_2') 81 | up9 = bilinear_up_and_concat( conv8, conv1, channel, channel*2, scope=ext+"g_up_4" ) 82 | conv9=slim.conv2d(up9, channel,[3,3], rate=1, activation_fn=lrelu, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_1') 83 | conv9=slim.conv2d(conv9,output_channel*4,[3,3], rate=1, activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer(),scope=ext+'g_conv9_2') 84 | return conv9 85 | 86 | 87 | config=tf.ConfigProto() 88 | config.gpu_options.allow_growth=True 89 | sess=tf.Session(config=config) 90 | input_i=tf.placeholder(tf.float32,shape=[1,None,None,2]) 91 | input_target=tf.placeholder(tf.float32,shape=[1,None,None,6]) 92 | 93 | with tf.variable_scope(tf.get_variable_scope()): 94 | with tf.variable_scope('individual'): 95 | g0=VCN(utils.build(tf.tile(input_i[:,:,:,0:1],[1,1,1,3])), reuse=False) 96 | g1=VCN(utils.build(tf.tile(input_i[:,:,:,1:2],[1,1,1,3])), reuse=True) 97 | 98 | saver=tf.train.Saver(max_to_keep=1000) 99 | sess.run([tf.global_variables_initializer()]) 100 | 101 | var_restore = [v for v in tf.trainable_variables()] 102 | saver_restore=tf.train.Saver(var_restore) 103 | ckpt=tf.train.get_checkpoint_state("pretrained_models/" + model) 104 | print('loaded '+ ckpt.model_checkpoint_path) 105 | saver_restore.restore(sess,ckpt.model_checkpoint_path) 106 | 107 | folder = video_path.split('/')[-1] 108 | output_dir = ARGS.video_path[:-1] + "_colorized" if ARGS.video_path.endswith("/") else ARGS.video_path + "_colorized" 109 | 110 | # if ARGS.output_dir is None: 111 | # output_dir = "test_result/{}/{}".format(model, folder) 112 | # else: 113 | # output_dir = ARGS.output_dir 114 | if img_path is None: 115 | img_names = utils.get_names(video_path) 116 | ind=0 117 | for frame_id, img_name in enumerate(img_names): 118 | im = np.array(Image.open(img_name).convert("L")) / 255. 119 | h = im.shape[0]//32*32 120 | w = im.shape[1]//32*32 121 | im = im[np.newaxis,:h,:w,np.newaxis] 122 | st=time.time() 123 | output=sess.run(g0,feed_dict={input_i:np.concatenate((im,im),axis=3)}) 124 | if frame_id % 10 == 0: 125 | print("test time for colorizing frame %s --> %.3f"%(ind, time.time()-st)) 126 | for idx in range(5): 127 | os.makedirs("{}/result{}".format(output_dir, idx), exist_ok=True) 128 | out_all = np.concatenate([output[:,:,:,3*i:3*i+3] for i in range(4)],axis=2) 129 | for idx in range(4): 130 | imageio.imwrite("{}/result{}/{:05d}.jpg".format(output_dir, idx+1, ind), 131 | np.uint8(np.maximum(np.minimum(output[0,:,:,3*idx:3*idx+3] * 255.0,255.0),0.0))) 132 | imageio.imwrite("{}/result0/{:05d}.jpg".format(output_dir, ind),np.uint8(np.maximum(np.minimum(out_all[0,:,:,:] * 255.0,255.0),0.0))) 133 | imageio.imwrite("{}/result0/input_{:05d}.jpg".format(output_dir, ind),np.uint8(np.maximum(np.minimum(im[0,:,:,0] * 255.0,255.0),0.0))) 134 | 135 | ind+=1 136 | 137 | else: 138 | im = np.array(Image.open(img_name).convert("L")) / 255. 139 | h=im.shape[0]//32*32 140 | w=im.shape[1]//32*32 141 | im=im[np.newaxis,:h,:w,np.newaxis] 142 | st=time.time() 143 | output=sess.run(g0,feed_dict={input_i:np.concatenate((im,im),axis=3)}) 144 | print("test time for frame %s --> %.3f"%(img_path, time.time()-st)) 145 | folder = video_path.split('/')[-1] 146 | out_all = np.concatenate([output[:,:,:,3*i:3*i+3] for i in range(4)],axis=2) 147 | for idx in range(4): 148 | imageio.imwrite("test_result/%s/%s_result%d.jpg"%(model,img_path.split('/')[-1][:-4],idx+1),np.uint8(np.maximum(np.minimum(output[0,:,:,3*idx:3*idx+3] * 255.0,255.0),0.0))) 149 | imageio.imwrite("test_result/%s/%s_result.jpg"%(model,img_path.split('/')[-1][:-4]),np.uint8(np.maximum(np.minimum(out_all[0,:,:,:] * 255.0,255.0),0.0))) 150 | imageio.imwrite("test_result/%s/%s_input.jpg"%(model,img_path.split('/')[-1][:-4]),np.uint8(np.maximum(np.minimum(im[0,:,:,0] * 255.0,255.0),0.0))) 151 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import os, cv2 4 | import math 5 | from scipy import io 6 | import myflowlib as flowlib 7 | import scipy.misc as sic 8 | # from flownet2.src import flow_warp 9 | from tensorflow.python.ops import array_ops 10 | from tensorflow.python.ops import math_ops 11 | from tensorflow.python.framework import ops 12 | 13 | 14 | IMG_EXTENSIONS = [ 15 | '.png', '.PNG', 'jpg', 'JPG', '.jpeg', '.JPEG', 16 | '.ppm', '.PPM', '.bmp', '.BMP', 17 | ] 18 | def check_image(image): 19 | assertion = tf.assert_equal(tf.shape(image)[-1], 3, message="image must have 3 color channels") 20 | with tf.control_dependencies([assertion]): 21 | image = tf.identity(image) 22 | 23 | if image.get_shape().ndims not in (3, 4): 24 | raise ValueError("image must be either 3 or 4 dimensions") 25 | 26 | # make the last dimension 3 so that you can unstack the colors 27 | shape = list(image.get_shape()) 28 | shape[-1] = 3 29 | image.set_shape(shape) 30 | return image 31 | 32 | def get_names(dir='./'): 33 | old_names = os.popen("ls %s"%dir).readlines() 34 | new_names = [None]*len(old_names) 35 | for idx in range(len(old_names)): 36 | new_names[idx] = dir+'/'+old_names[idx][:-1] 37 | return new_names 38 | 39 | 40 | def is_image_file(filename): 41 | return any(filename.endswith(extension) for extension in IMG_EXTENSIONS) 42 | 43 | ''' 44 | Return: gray_images, color_images 45 | gray_images- [num_frames, H, W, 1] 46 | color_images- [num_frames, H, W, 3] 47 | pixel values [0,1] 48 | ''' 49 | def read_image_sequence(filename, num_frames): 50 | file1 = os.path.splitext(os.path.basename(filename))[0] 51 | ext = os.path.splitext(os.path.basename(filename))[1] 52 | try: 53 | img1 = sic.imread(filename).astype(np.float32) 54 | imgh1 = img1 55 | except: 56 | #print("Cannot read the first frame.") 57 | return None, None 58 | if len(img1.shape) == 2: # discard grayscale images 59 | return None, None 60 | 61 | img1 = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY) 62 | img1 = np.expand_dims(img1,2) 63 | 64 | img_l_seq=img1/255.0 65 | img_h_seq=imgh1/255.0 66 | for i in range(num_frames-1): 67 | filei = int(file1) + i + 1 68 | filenamei = os.path.split(filename)[0] + "/" + "{:>05}".format(filei).format() + ext 69 | try: 70 | imgi = sic.imread(filenamei).astype(np.float32) 71 | imghi = imgi 72 | except: 73 | #print("Cannot read the following %d frames\n"%(num_frames)) 74 | return None, None 75 | imgi = cv2.cvtColor(imgi, cv2.COLOR_RGB2GRAY) 76 | imgi = np.expand_dims(imgi,2) 77 | 78 | img_l_seq = np.concatenate((img_l_seq,imgi/255.0),axis=2) 79 | img_h_seq = np.concatenate((img_h_seq,imghi/255.0),axis=2) 80 | 81 | return img_l_seq, img_h_seq 82 | 83 | 84 | def read_image_SPMC(filename, num_frames): 85 | file1 = os.path.splitext(os.path.basename(filename))[0] 86 | ext = os.path.splitext(os.path.basename(filename))[1] 87 | 88 | img1 = sic.imread(filename) 89 | imgh1 = sic.imread(filename.replace("input4","gt")) 90 | if img1 is None or imgh1 is None: 91 | print("Cannot read the first frame\n") 92 | return None,None 93 | if len(img1.shape) == 2: 94 | isgray=True 95 | img1 = np.concatenate((img1,img1,img1),axis=2) 96 | imgh1 = np.concatenate((imgh1,imgh1,imgh1),axis=2) 97 | else: 98 | isgray=False 99 | 100 | img_l_seq=img1 101 | img_h_seq=imgh1 102 | for i in range(num_frames-1): 103 | filei = int(file1) + i + 1 104 | filenamei = os.path.split(filename)[0] + "/" + "{:>04}".format(filei).format() + ext 105 | imgi = sic.imread(filenamei, -1) 106 | imghi = sic.imread(filenamei.replace("input4","gt"), -1) 107 | if imgi is None: 108 | print("Cannot read the following %d frames\n"%(num_frames)) 109 | return None,None 110 | else: 111 | if isgray: 112 | imgi = np.concatenate((imgi,imgi,imgi),axis=2) 113 | imghi = np.concatenate((imghi,imghi,imghi),axis=2) 114 | img_l_seq = np.concatenate((img_l_seq,imgi),axis=2) 115 | img_h_seq = np.concatenate((img_h_seq,imghi),axis=2) 116 | 117 | return img_l_seq, img_h_seq 118 | 119 | def read_flow_sequence_w_mask(filename, num_frames): 120 | file1 = os.path.splitext(os.path.basename(filename))[0] 121 | folder = os.path.split(filename)[0] 122 | ext = os.path.splitext(os.path.basename(filename))[1] 123 | 124 | filej = file1 125 | for i in range(num_frames-1): 126 | filei = int(file1) + i + 1 127 | if "SPMC" in filename: 128 | flow_forward = flowlib.read_flow(folder+"/Forward/{:>04}".format(filej).format()+"_"+"{:>04}".format(filei).format()+".flo") 129 | flow_forward_mask = sic.imread(folder+"/Forward/{:>04}".format(filej).format()+"_"+"{:>04}".format(filei).format()+".png") 130 | flow_backward = flowlib.read_flow(folder+"/Backward/{:>04}".format(filei).format()+"_"+"{:>04}".format(filej).format()+".flo") 131 | flow_backward_mask = sic.imread(folder+"/Forward/{:>04}".format(filej).format()+"_"+"{:>04}".format(filei).format()+".png") 132 | else: 133 | flow_forward = flowlib.read_flow(folder.replace("480p","Forward")+"/"+"{:>05}".format(filej).format()+"_"+"{:>05}".format(filei).format()+".flo") 134 | flow_forward_mask = sic.imread(folder.replace("480p","Forward")+"/"+"{:>05}".format(filej).format()+"_"+"{:>05}".format(filei).format()+".png") 135 | flow_backward = flowlib.read_flow(folder.replace("480p","Backward")+"/"+"{:>05}".format(filei).format()+"_"+"{:>05}".format(filej).format()+".flo") 136 | flow_backward_mask = sic.imread(folder.replace("480p","Backward")+"/"+"{:>05}".format(filei).format()+"_"+"{:>05}".format(filej).format()+".png") 137 | filej = filei 138 | if i == 0: 139 | flow_forward_seq = flow_forward 140 | flow_backward_seq = flow_backward 141 | flow_backward_mask_seq = np.expand_dims(flow_backward_mask, axis=2) 142 | flow_forward_mask_seq = np.expand_dims(flow_forward_mask, axis=2) 143 | else: 144 | flow_forward_seq = np.concatenate((flow_forward_seq, flow_forward), axis=2) 145 | flow_backward_seq = np.concatenate((flow_backward_seq, flow_backward), axis=2) 146 | flow_forward_mask_seq = np.concatenate((flow_forward_mask_seq, np.expand_dims(flow_forward_mask, axis=2)), axis=2) 147 | flow_backward_mask_seq = np.concatenate((flow_backward_mask_seq, np.expand_dims(flow_backward_mask, axis=2)), axis=2) 148 | 149 | return flow_forward_seq, flow_backward_seq, flow_forward_mask_seq, flow_backward_mask_seq 150 | 151 | 152 | 153 | 154 | def read_flow_sequence(filename, num_frames): 155 | file1 = os.path.splitext(os.path.basename(filename))[0] 156 | folder = os.path.split(filename)[0] 157 | ext = os.path.splitext(os.path.basename(filename))[1] 158 | 159 | filej = file1 160 | for i in range(num_frames-1): 161 | filei = int(file1) + i + 1 162 | if "SPMC" in filename: 163 | flow_forward = flowlib.read_flow(folder+"/Forward/{:>04}".format(filej).format()+"_"+"{:>04}".format(filei).format()+".flo") 164 | flow_backward = flowlib.read_flow(folder+"/Backward/{:>04}".format(filei).format()+"_"+"{:>04}".format(filej).format()+".flo") 165 | else: 166 | flow_forward = flowlib.read_flow(folder.replace("480p","Forward")+"/"+"{:>05}".format(filej).format()+"_"+"{:>05}".format(filei).format()+".flo") 167 | flow_backward = flowlib.read_flow(folder.replace("480p","Backward")+"/"+"{:>05}".format(filei).format()+"_"+"{:>05}".format(filej).format()+".flo") 168 | filej = filei 169 | if i == 0: 170 | flow_forward_seq = flow_forward 171 | flow_backward_seq = flow_backward 172 | else: 173 | flow_forward_seq = np.concatenate((flow_forward_seq, flow_forward), axis=2) 174 | flow_backward_seq = np.concatenate((flow_backward_seq, flow_backward), axis=2) 175 | 176 | return flow_forward_seq, flow_backward_seq 177 | 178 | 179 | def read_flow_sintel(filename, num_frames, substr="clean"): 180 | file1 = os.path.splitext(os.path.basename(filename))[0] 181 | folder = os.path.split(filename)[0] 182 | flowfolder = folder.replace(substr, "flow") 183 | maskfolder = folder.replace(substr, "occlusions") 184 | invalfolder = folder.replace(substr, "invalid") 185 | start_frame = int(file1.split("_")[1]) 186 | for i in range(num_frames-1): 187 | # name = "/frame_" + "{:>04}".format(str(start_frame + i )).format() 188 | name = "/frame_" + "{:>04}".format(str(start_frame + i - 1)).format() 189 | # print("flow file: ",name) 190 | try: 191 | flowi = flowlib.read_flow(flowfolder+name+".flo") 192 | flow_maski = sic.imread(maskfolder+name+".png",0).astype(np.float32)/255. 193 | flow_invali = sic.imread(invalfolder+name+".png",0).astype(np.float32)/255. 194 | 195 | flow_maski = np.expand_dims((flow_maski),axis=2) 196 | flow_invali = np.expand_dims((flow_invali),axis=2) 197 | except: 198 | return None, None, None 199 | if i == 0: 200 | # initialize the sequence 201 | flow_seq = flowi 202 | flow_seq_mask = flow_maski 203 | flow_seq_inval = flow_invali 204 | else: 205 | flow_seq = np.concatenate((flow_seq, flowi),axis=2) 206 | flow_seq_mask = np.concatenate((flow_seq_mask, flow_maski), axis=2) 207 | flow_seq_inval = np.concatenate((flow_seq_inval, flow_invali), axis=2) 208 | return flow_seq, flow_seq_mask, flow_seq_inval 209 | 210 | def read_image_path(file_path): 211 | path_all=[] 212 | for dirname in file_path: 213 | for root, dir, fnames in sorted(os.walk(dirname)): 214 | for fname in sorted(fnames): 215 | if is_image_file(fname): 216 | path_all.append(os.path.join(root, fname)) 217 | return path_all 218 | 219 | def flip_images(X): 220 | num_img = X.shape[0] 221 | magic=np.random.random() 222 | if magic < 0.3: 223 | for i in range(num_img): 224 | X[i,...] = np.fliplr(X[i,...]) 225 | magic=np.random.random() 226 | if magic < 0.3: 227 | for i in range(num_img): 228 | X[i,...] = np.fliplr(X[i,...]) 229 | return X 230 | 231 | def pad_images(X,a): 232 | num_img = X.shape[0] 233 | h_orig,w_orig = X.shape[1:3] 234 | newX = np.ones((num_img,a,a,3)) 235 | for i in range(num_img): 236 | pad_width=((0,a-h_orig),(0,a-w_orig),(0,0)) 237 | newX[i,...] = np.pad(X[i,...],pad_width,'constant') 238 | return newX 239 | 240 | def crop_images(X,a,b,is_sq=False): 241 | h_orig,w_orig = X.shape[1:3] 242 | w_crop = np.random.randint(a, b) 243 | r = w_crop/w_orig 244 | h_crop = np.int(h_orig*r) 245 | try: 246 | w_offset = np.random.randint(0, w_orig-w_crop-1) 247 | h_offset = np.random.randint(0, h_orig-h_crop-1) 248 | except: 249 | print("Original W %d, desired W %d"%(w_orig,w_crop)) 250 | print("Original H %d, desired H %d"%(h_orig,h_crop)) 251 | return X[:,h_offset:h_offset+h_crop-1,w_offset:w_offset+w_crop-1,:] 252 | 253 | def degamma(X): 254 | return np.power(X, 2.2) 255 | 256 | def gamma(X): 257 | return np.power(X, 1/2.2) 258 | 259 | def build_net(ntype,nin,nwb=None,name=None): 260 | if ntype=='conv': 261 | return tf.nn.relu(tf.nn.conv2d(nin,nwb[0],strides=[1,1,1,1],padding='SAME',name=name)+nwb[1]) 262 | elif ntype=='pool': 263 | return tf.nn.avg_pool(nin,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME') 264 | 265 | 266 | def get_weight_bias(vgg_layers,i): 267 | weights=vgg_layers[i][0][0][2][0][0] 268 | weights=tf.constant(weights) 269 | bias=vgg_layers[i][0][0][2][0][1] 270 | bias=tf.constant(np.reshape(bias,(bias.size))) 271 | return weights,bias 272 | 273 | vgg_rawnet=io.loadmat('VGG_Model/imagenet-vgg-verydeep-19.mat') 274 | def build_vgg19(input,reuse=False): 275 | with tf.variable_scope("vgg19"): 276 | if reuse: 277 | tf.get_variable_scope().reuse_variables() 278 | net={} 279 | vgg_layers=vgg_rawnet['layers'][0] 280 | net['input']=input-np.array([123.6800, 116.7790, 103.9390]).reshape((1,1,1,3)) 281 | net['conv1_1']=build_net('conv',net['input'],get_weight_bias(vgg_layers,0),name='vgg_conv1_1') 282 | net['conv1_2']=build_net('conv',net['conv1_1'],get_weight_bias(vgg_layers,2),name='vgg_conv1_2') 283 | net['pool1']=build_net('pool',net['conv1_2']) 284 | net['conv2_1']=build_net('conv',net['pool1'],get_weight_bias(vgg_layers,5),name='vgg_conv2_1') 285 | net['conv2_2']=build_net('conv',net['conv2_1'],get_weight_bias(vgg_layers,7),name='vgg_conv2_2') 286 | net['pool2']=build_net('pool',net['conv2_2']) 287 | net['conv3_1']=build_net('conv',net['pool2'],get_weight_bias(vgg_layers,10),name='vgg_conv3_1') 288 | net['conv3_2']=build_net('conv',net['conv3_1'],get_weight_bias(vgg_layers,12),name='vgg_conv3_2') 289 | net['conv3_3']=build_net('conv',net['conv3_2'],get_weight_bias(vgg_layers,14),name='vgg_conv3_3') 290 | net['conv3_4']=build_net('conv',net['conv3_3'],get_weight_bias(vgg_layers,16),name='vgg_conv3_4') 291 | net['pool3']=build_net('pool',net['conv3_4']) 292 | net['conv4_1']=build_net('conv',net['pool3'],get_weight_bias(vgg_layers,19),name='vgg_conv4_1') 293 | net['conv4_2']=build_net('conv',net['conv4_1'],get_weight_bias(vgg_layers,21),name='vgg_conv4_2') 294 | net['conv4_3']=build_net('conv',net['conv4_2'],get_weight_bias(vgg_layers,23),name='vgg_conv4_3') 295 | net['conv4_4']=build_net('conv',net['conv4_3'],get_weight_bias(vgg_layers,25),name='vgg_conv4_4') 296 | net['pool4']=build_net('pool',net['conv4_4']) 297 | net['conv5_1']=build_net('conv',net['pool4'],get_weight_bias(vgg_layers,28),name='vgg_conv5_1') 298 | net['conv5_2']=build_net('conv',net['conv5_1'],get_weight_bias(vgg_layers,30),name='vgg_conv5_2') 299 | return net 300 | 301 | def build(input): 302 | vgg19_features=build_vgg19(input[:,:,:,0:3]*255.0) 303 | for layer_id in range(1,6):#6 304 | vgg19_f = vgg19_features['conv%d_2'%layer_id] 305 | input = tf.concat([tf.image.resize_bilinear(vgg19_f,(tf.shape(input)[1],tf.shape(input)[2]))/255.0,input], axis=3) 306 | return input 307 | 308 | def build_nlayer(input, nlayer): 309 | vgg19_features=build_vgg19(input[:,:,:,0:3]*255.0) 310 | for layer_id in range(1,nlayer):#6 311 | vgg19_f = vgg19_features['conv%d_2'%layer_id] 312 | input = tf.concat([tf.image.resize_bilinear(vgg19_f,(tf.shape(input)[1],tf.shape(input)[2]))/255.0,input], axis=3) 313 | return input 314 | 315 | 316 | def linear(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 317 | shape = input_.get_shape().as_list() 318 | 319 | with tf.variable_scope(scope or "Linear"): 320 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 321 | tf.random_normal_initializer(stddev=stddev)) 322 | bias = tf.get_variable("bias", [output_size], 323 | initializer=tf.constant_initializer(bias_start)) 324 | if with_w: 325 | return tf.matmul(input_, matrix) + bias, matrix, bias 326 | else: 327 | return tf.matmul(input_, matrix) + bias 328 | def conv_2d(input_, output_dim, 329 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 330 | name="conv2d"): 331 | with tf.variable_scope(name): 332 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 333 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 334 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 335 | 336 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 337 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 338 | 339 | return conv 340 | -------------------------------------------------------------------------------- /video_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from glob import glob 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | from imageio import imread, imsave 7 | from os.path import isfile, join 8 | import cv2 9 | from moviepy.editor import * 10 | 11 | parser = argparse.ArgumentParser() 12 | 13 | parser.add_argument("--video_dir", default="demo.mkv", help="choose the dir of video") 14 | parser.add_argument("--out_frames_dir", default="demo.mkv", help="choose the dir of video") 15 | 16 | parser.add_argument("--colorized_frames_dir", default="demo.mkv", help="choose the dir of video") 17 | parser.add_argument("--colorized_video_dir", default="demo.mkv", help="choose the dir of video") 18 | 19 | parser.add_argument("--video2frames", action="store_true", help="convert video to frames") 20 | parser.add_argument("--frames2video", action="store_true", help="convert frames to video") 21 | parser.add_argument("--add_sound", action="store_true", help="convert video to frames") 22 | 23 | parser.add_argument("--fps", default=30, help="convert frames to video") 24 | 25 | 26 | ARGS = parser.parse_args() 27 | 28 | 29 | def video2frames(video_dir, out_frames_dir="None"): 30 | os.makedirs(out_frames_dir, exist_ok=True) 31 | video = VideoFileClip(video_dir) 32 | audio = video.audio 33 | audio.write_audiofile(out_frames_dir + ".mp3") 34 | 35 | vidcap = cv2.VideoCapture(video_dir) 36 | # Find OpenCV version 37 | (major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.') 38 | 39 | # With webcam get(CV_CAP_PROP_FPS) does not work. 40 | # Let's see for ourselves. 41 | 42 | if int(major_ver) < 3 : 43 | fps = vidcap.get(cv2.cv.CV_CAP_PROP_FPS) 44 | print("Frames per second using video.get(cv2.cv.CV_CAP_PROP_FPS): {0}".format(fps)) 45 | else : 46 | fps = vidcap.get(cv2.CAP_PROP_FPS) 47 | print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps)) 48 | success,image = vidcap.read() 49 | count = 0 50 | success = True 51 | while success: 52 | success,image = vidcap.read() 53 | if cv2.waitKey(10) == 27: # exit if Escape is hit 54 | break 55 | if image is None: 56 | print("Fps is {}".format(fps)) 57 | return 0 58 | if count % 100 == 0: 59 | print("Video to frames: {}/frame{:04d}.png Image shape:" .format(out_frames_dir, count), image.shape) 60 | cv2.imwrite("{}/frame{:04d}.png".format(out_frames_dir, count), image) # save frame as JPEG file 61 | count += 1 62 | vidcap.release() 63 | audio.release() 64 | print("Fps is {}".format(fps)) 65 | return int(fps) 66 | 67 | 68 | def frames2video(frames_dir= './images/testing/', colorized_video_dir = 'output.mp4', fps=24): 69 | frame_array = [] 70 | files = sorted(glob(frames_dir+"/*")) 71 | print(frames_dir, len(files)) 72 | for i in range(len(files)): 73 | filename= files[i] 74 | #reading each files 75 | if i % 100 == 0: 76 | print("frames2video:", filename) 77 | img = cv2.imread(filename) 78 | height, width, layers = img.shape 79 | size = (width,height) 80 | frame_array.append(img) #inserting the frames into an image array 81 | out = cv2.VideoWriter(colorized_video_dir, cv2.VideoWriter_fourcc(*'DIVX'), fps, size) 82 | for i in range(len(frame_array)): 83 | out.write(frame_array[i]) 84 | out.release() 85 | 86 | def add_mp3(source_video, target_video): 87 | fg_video = VideoFileClip(source_video) 88 | video = VideoFileClip(target_video) 89 | audio = fg_video.audio 90 | videoclip2 = video.set_audio(audio) 91 | name1 = target_video.split('.', 1)[0] + "_withsound.mp4" 92 | videoclip2.write_videofile(name1) 93 | 94 | 95 | if ARGS.video2frames: 96 | fps = video2frames(ARGS.video_dir, ARGS.out_frames_dir) 97 | # print("fps is {}".format(fps)) 98 | 99 | if ARGS.frames2video: 100 | frames2video(ARGS.colorized_frames_dir, ARGS.colorized_video_dir, ARGS.fps) 101 | print(ARGS.colorized_video_dir) 102 | 103 | if ARGS.add_sound: 104 | add_mp3(ARGS.video_dir, ARGS.colorized_video_dir) --------------------------------------------------------------------------------