├── .github
└── workflows
│ └── test.yml
├── .gitignore
├── .gitmodules
├── CMakeLists.txt
├── Demo.ipynb
├── LICENSE
├── README.md
├── colorization
├── __init__.py
├── baseline
│ ├── __init__.py
│ ├── colorizer.py
│ └── utils.py
├── include
│ └── colorization.h
├── iterative_colorizer
│ ├── __init__.py
│ ├── iterative_colorizer.py
│ ├── utils.py
│ └── window_neighbour.py
└── main.cpp
├── colorize.py
├── data
├── original
│ ├── example.png
│ ├── example2.png
│ ├── example3.png
│ ├── example4.png
│ ├── example5.png
│ ├── example6.png
│ ├── example7.png
│ └── example8.png
├── results
│ ├── result.png
│ ├── result2.png
│ ├── result3.png
│ ├── result4.png
│ ├── result5.png
│ ├── result6.png
│ ├── result7.png
│ └── result8.png
└── visual-clues
│ ├── example.png
│ ├── example2_marked.png
│ ├── example3_marked.png
│ ├── example4.png
│ ├── example5.png
│ ├── example6.png
│ ├── example7.png
│ └── example8.png
├── install.sh
├── requirements.txt
└── tests.py
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: test
2 | on:
3 | push:
4 | branches:
5 | - master
6 | - feature/colorizer
7 | - feature/cpp-implementation
8 | paths:
9 | - .github/**
10 | - colorization/**
11 | - tests.py
12 | - requirements.txt
13 | jobs:
14 | testing-on-linux:
15 | runs-on: ubuntu-latest
16 | steps:
17 | - uses: actions/checkout@v2
18 | - run: |
19 | pip install -U pip
20 | pip install -r requirements.txt
21 | pytest tests.py -s
22 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | venv/
2 | .idea/
3 | build/
4 | .vscode/
5 | **pycache**
6 | .ipynb_checkpoints/
7 | cmake-build-debug/
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "colorization/include/cpptqdm"]
2 | path = colorization/include/cpptqdm
3 | url = https://github.com/aminnj/cpptqdm
4 |
--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | cmake_minimum_required(VERSION 3.19)
2 | project(colorization)
3 | set(CMAKE_CXX_STANDARD 14)
4 |
5 | find_package(Eigen3 REQUIRED)
6 | include_directories(${EIGEN3_INCLUDE_DIR})
7 |
8 | find_package(OpenCV REQUIRED)
9 | include_directories(${OpenCV_INCLUDE_DIRS})
10 |
11 | include_directories(${PROJECT_NAME} ${CMAKE_SOURCE_DIR}/colorization/include)
12 |
13 | add_executable(${PROJECT_NAME} colorization/main.cpp colorization/include/colorization.h)
14 | target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS})
15 |
--------------------------------------------------------------------------------
/Demo.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "e6deadda",
6 | "metadata": {
7 | "colab_type": "text",
8 | "id": "view-in-github"
9 | },
10 | "source": [
11 | ""
12 | ]
13 | },
14 | {
15 | "cell_type": "markdown",
16 | "id": "Mrgh7NHkUNAo",
17 | "metadata": {
18 | "id": "Mrgh7NHkUNAo"
19 | },
20 | "source": [
21 | "## Clone the repository"
22 | ]
23 | },
24 | {
25 | "cell_type": "code",
26 | "execution_count": null,
27 | "id": "gMeivfsWUMtQ",
28 | "metadata": {
29 | "colab": {
30 | "base_uri": "https://localhost:8080/"
31 | },
32 | "id": "gMeivfsWUMtQ",
33 | "outputId": "09c4990d-67c7-4063-bfa0-c58445e69866"
34 | },
35 | "outputs": [],
36 | "source": [
37 | "!git clone https://github.com/soumik12345/colorization-using-optimization\n",
38 | "%cd colorization-using-optimization"
39 | ]
40 | },
41 | {
42 | "cell_type": "markdown",
43 | "id": "17f836a3",
44 | "metadata": {
45 | "id": "17f836a3"
46 | },
47 | "source": [
48 | "## Baseline Colorizer"
49 | ]
50 | },
51 | {
52 | "cell_type": "code",
53 | "execution_count": null,
54 | "id": "47540fb4",
55 | "metadata": {
56 | "id": "47540fb4"
57 | },
58 | "outputs": [],
59 | "source": [
60 | "from colorization import Colorizer"
61 | ]
62 | },
63 | {
64 | "cell_type": "code",
65 | "execution_count": null,
66 | "id": "fb01dd85",
67 | "metadata": {
68 | "id": "fb01dd85"
69 | },
70 | "outputs": [],
71 | "source": [
72 | "colorizer = Colorizer(\n",
73 | " gray_image_file='./data/original/example.png',\n",
74 | " visual_clues_file='./data/visual-clues/example.png'\n",
75 | ")"
76 | ]
77 | },
78 | {
79 | "cell_type": "code",
80 | "execution_count": null,
81 | "id": "21fcde2e",
82 | "metadata": {
83 | "colab": {
84 | "base_uri": "https://localhost:8080/",
85 | "height": 298
86 | },
87 | "id": "21fcde2e",
88 | "outputId": "892a6982-1ffe-4cbb-c687-d4cb38271681"
89 | },
90 | "outputs": [],
91 | "source": [
92 | "colorizer.plot_inputs()"
93 | ]
94 | },
95 | {
96 | "cell_type": "code",
97 | "execution_count": null,
98 | "id": "7bd9428a",
99 | "metadata": {
100 | "colab": {
101 | "base_uri": "https://localhost:8080/"
102 | },
103 | "id": "7bd9428a",
104 | "outputId": "89963492-5ad3-46bf-d83e-a634eca1b665"
105 | },
106 | "outputs": [],
107 | "source": [
108 | "%%time\n",
109 | "result = colorizer.colorize()"
110 | ]
111 | },
112 | {
113 | "cell_type": "code",
114 | "execution_count": null,
115 | "id": "c8648e42",
116 | "metadata": {
117 | "colab": {
118 | "base_uri": "https://localhost:8080/",
119 | "height": 386
120 | },
121 | "id": "c8648e42",
122 | "outputId": "f6587c81-d824-4af8-b82a-57fde8c32e83"
123 | },
124 | "outputs": [],
125 | "source": [
126 | "colorizer.plot_results(result)"
127 | ]
128 | },
129 | {
130 | "cell_type": "markdown",
131 | "id": "17e517b1",
132 | "metadata": {
133 | "id": "17e517b1"
134 | },
135 | "source": [
136 | "## Iterative Colorizer"
137 | ]
138 | },
139 | {
140 | "cell_type": "code",
141 | "execution_count": null,
142 | "id": "48e35f19",
143 | "metadata": {
144 | "id": "48e35f19"
145 | },
146 | "outputs": [],
147 | "source": [
148 | "from colorization import IterativeColorizer"
149 | ]
150 | },
151 | {
152 | "cell_type": "code",
153 | "execution_count": null,
154 | "id": "67dac6b5",
155 | "metadata": {
156 | "id": "67dac6b5"
157 | },
158 | "outputs": [],
159 | "source": [
160 | "colorizer = IterativeColorizer(\n",
161 | " original_image='./data/original/example.png',\n",
162 | " visual_clues='./data/visual-clues/example.png'\n",
163 | ")"
164 | ]
165 | },
166 | {
167 | "cell_type": "code",
168 | "execution_count": null,
169 | "id": "48734ad9",
170 | "metadata": {
171 | "colab": {
172 | "base_uri": "https://localhost:8080/",
173 | "height": 298
174 | },
175 | "id": "48734ad9",
176 | "outputId": "1d14f808-f2f2-4e9c-c87b-95f3172bfea6"
177 | },
178 | "outputs": [],
179 | "source": [
180 | "colorizer.plot_inputs()"
181 | ]
182 | },
183 | {
184 | "cell_type": "code",
185 | "execution_count": null,
186 | "id": "02056cde",
187 | "metadata": {
188 | "colab": {
189 | "base_uri": "https://localhost:8080/"
190 | },
191 | "id": "02056cde",
192 | "outputId": "491a83dc-e87d-46fb-f4f3-648bb7756c9e"
193 | },
194 | "outputs": [],
195 | "source": [
196 | "%%time\n",
197 | "colorizer.colorize(epochs=600, log_interval=100)"
198 | ]
199 | },
200 | {
201 | "cell_type": "code",
202 | "execution_count": null,
203 | "id": "8a74de03",
204 | "metadata": {
205 | "colab": {
206 | "base_uri": "https://localhost:8080/",
207 | "height": 1000
208 | },
209 | "id": "8a74de03",
210 | "outputId": "9104deb8-4089-4e0e-ad10-04bd7a636bd6"
211 | },
212 | "outputs": [],
213 | "source": [
214 | "colorizer.plot_results(log_interval=100)"
215 | ]
216 | },
217 | {
218 | "cell_type": "code",
219 | "execution_count": null,
220 | "id": "jdB19ikpX1Or",
221 | "metadata": {
222 | "id": "jdB19ikpX1Or"
223 | },
224 | "outputs": [],
225 | "source": []
226 | }
227 | ],
228 | "metadata": {
229 | "colab": {
230 | "include_colab_link": true,
231 | "name": "Demo.ipynb",
232 | "provenance": []
233 | },
234 | "interpreter": {
235 | "hash": "556b822321b97cfa85b8ef545cd8388f5d22aa515c082e37848490687a34f054"
236 | },
237 | "kernelspec": {
238 | "display_name": "Python 3 (ipykernel)",
239 | "language": "python",
240 | "name": "python3"
241 | },
242 | "language_info": {
243 | "codemirror_mode": {
244 | "name": "ipython",
245 | "version": 3
246 | },
247 | "file_extension": ".py",
248 | "mimetype": "text/x-python",
249 | "name": "python",
250 | "nbconvert_exporter": "python",
251 | "pygments_lexer": "ipython3",
252 | "version": "3.8.10"
253 | }
254 | },
255 | "nbformat": 4,
256 | "nbformat_minor": 5
257 | }
258 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 Soumik Rakshit
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Colorization using Optimization
2 |
3 |
4 |
5 | Python and C++ implementations of a user-guided image/video colorization technique as proposed by the paper
6 | [Colorization Using Optimization](https://dl.acm.org/doi/10.1145/1015706.1015780). The algorithm is based on a simple premise; neighboring pixels in space-time that have similar intensities should have similar colors. This premise is formalized using a quadratic cost function that obtains an optimization problem that can be solved efficiently using standard techniques. **While using this alogorithm, an artist only needs to annotate the image with a few color scribbles or visual clues, and the indicated colors are automatically propagated in both space and time to produce a fully colorized image or sequence.** The annotation can be done using any drawing tool such as [JSPaint](https://jspaint.app/) or [Gimp](https://www.gimp.org/).
7 |
8 | ## Instructions
9 |
10 | ### Instructions for running python version
11 |
12 | 1. Create a virtualenv using:
13 | - `virtualenv venv --python=python3`
14 | - `source venv/bin/activate`
15 | - `pip install -r requirements.txt`
16 |
17 | 2. Colorize images using the CLI:
18 | ```
19 | python colorize.py
20 |
21 | Options:
22 | --original_image TEXT Original Image Path
23 | --visual_clue TEXT Visual Clue Image Path
24 | --result_path TEXT Colorized Image Path (without file extensions)
25 | -i, --use_itercative Use Iterative Mode
26 | --epochs INTEGER Number of epochs for Iterative Mode
27 | --log_intervals INTEGER Log Interval
28 | --help Show this message and exit.
29 | ```
30 |
31 | 3. Alternatively, you can run on Google Colab using
32 |
33 | ### Instructions to build C++ version
34 |
35 | 1. Install dependencies using `sh install.sh`
36 |
37 | 2. Create a build directory `mkdir build && cd build`
38 |
39 | 3. Generate makefiles and compile using `cmake .. && make`
40 |
41 | 4. Run the executable using `./colorization [input-image] [visual-clues] [result] [gamma] [threshold]`
42 |
43 | 5. Alternatively, you can download the executable from [here](https://github.com/soumik12345/colorization-using-optimization/releases/download/0.1/colorization) and run it (installation of dependencies is still needed).
44 |
45 | ## Results
46 |
47 |
Original Image | 50 |Visual Clues | 51 |Colorized Image | 52 |
---|---|---|
![]() |
55 | ![]() |
56 | ![]() |
57 |
![]() |
60 | ![]() |
61 | ![]() |
62 |
![]() |
65 | ![]() |
66 | ![]() |
67 |
![]() |
70 | ![]() |
71 | ![]() |
72 |
![]() |
75 | ![]() |
76 | ![]() |
77 |
![]() |
80 | ![]() |
81 | ![]() |
82 |
![]() |
85 | ![]() |
86 | ![]() |
87 |
![]() |
90 | ![]() |
91 | ![]() |
92 |