├── .gitignore
├── CHANGELOG.md
├── CMakeLists.txt
├── Equi2Rect.cpp
├── Equi2Rect.hpp
├── LICENSE
├── README.md
├── config.yaml
├── images
    ├── pano.jpg
    └── pano_rect.jpg
└── main.cpp


/.gitignore:
--------------------------------------------------------------------------------
1 | build/


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | ### 06/21/2022
 4 | - Added changelog
 5 | - Further refactoring & modernizing (removing macros, using proper hpp and cpp filestructure)
 6 | - Added config file to change viewport settings on the fly
 7 | - New dependency: yaml-cpp for parsing the config file
 8 | - Bugfix: pan angle is now semantically correct
 9 | 
10 | ### 11/23/2019
11 | - Code was refactored a bit to make it much more readable and compatible to newer OpenCV versions 
12 | - Still needs optimization though


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.10)
 2 | project(equirectangular2rectlinear)
 3 | 
 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 5 | find_package(OpenCV REQUIRED )
 6 | find_package(yaml-cpp REQUIRED)
 7 | 
 8 | add_library(Equi2Rect
 9 |             Equi2Rect.hpp
10 |             Equi2Rect.cpp)
11 | 
12 | target_link_libraries(Equi2Rect 
13 |                     ${OpenCV_LIBS}
14 |                     yaml-cpp )
15 | 
16 | 
17 | add_executable(equi2rect_example main.cpp)
18 | 
19 | target_link_libraries(equi2rect_example Equi2Rect)
20 | 


--------------------------------------------------------------------------------
/Equi2Rect.cpp:
--------------------------------------------------------------------------------
  1 | #include "Equi2Rect.hpp"
  2 | #include <yaml-cpp/yaml.h>
  3 | #include <math.h>
  4 | 
  5 | Equi2Rect::Equi2Rect()
  6 | {
  7 |     YAML::Node config = YAML::LoadFile("config.yaml");
  8 | 
  9 |     // viewport size
 10 |     viewport.width = config["viewport"]["width"].as<int>();
 11 |     viewport.height = config["viewport"]["height"].as<int>();
 12 | 
 13 |     // specify viewing direction
 14 |     viewport.pan_angle = config["viewport"]["pan_angle"].as<double>();
 15 |     viewport.tilt_angle = config["viewport"]["tilt_angle"].as<double>();
 16 |     viewport.roll_angle = config["viewport"]["roll_angle"].as<double>();
 17 | 
 18 |     // create rotation matrix
 19 |     Rot = eul2rotm(viewport.tilt_angle, viewport.pan_angle, viewport.roll_angle);
 20 | 
 21 |     // specify focal length of the final pinhole image
 22 |     focal_length = config["camera"]["focal_length"].as<int>();
 23 |     ;
 24 | 
 25 |     // create camera matrix K
 26 |     K = (cv::Mat_<double>(3, 3) << focal_length, 0, viewport.width / 2,
 27 |          0, focal_length, viewport.height / 2,
 28 |          0, 0, 1);
 29 | 
 30 |     // read src image
 31 |     img_src = cv::imread(config["files"]["img_src_path"].as<std::string>(), cv::IMREAD_COLOR);
 32 |     if (img_src.empty())
 33 |     {
 34 |         throw std::invalid_argument("Error: Could not load image!");
 35 |     }
 36 | 
 37 |     // initialize result image
 38 |     img_dst = cv::Mat(viewport.height, viewport.width, CV_8UC3, cv::Scalar(0, 0, 0));
 39 |     img_dst_path = config["files"]["img_dst_path"].as<std::string>();
 40 | }
 41 | 
 42 | auto Equi2Rect::save_rectlinear_image() -> void
 43 | {
 44 |     this->bilinear_interpolation();
 45 |     cv::imwrite(img_dst_path, img_dst);
 46 | }
 47 | 
 48 | auto Equi2Rect::show_rectlinear_image() -> void
 49 | {
 50 |     cv::namedWindow("projected image", cv::WINDOW_AUTOSIZE);
 51 |     cv::imshow("projected image", img_dst);
 52 |     cv::waitKey(0);
 53 |     cv::destroyWindow("projected image");
 54 | }
 55 | 
 56 | auto Equi2Rect::eul2rotm(double rotx, double roty, double rotz) -> cv::Mat
 57 | {
 58 | 
 59 |     cv::Mat R_x = (cv::Mat_<double>(3, 3) << 1, 0, 0,
 60 |                    0, cos(rotx), -sin(rotx),
 61 |                    0, sin(rotx), cos(rotx));
 62 | 
 63 |     cv::Mat R_y = (cv::Mat_<double>(3, 3) << cos(roty), 0, sin(roty),
 64 |                    0, 1, 0,
 65 |                    -sin(roty), 0, cos(roty));
 66 | 
 67 |     cv::Mat R_z = (cv::Mat_<double>(3, 3) << cos(rotz), -sin(rotz), 0,
 68 |                    sin(rotz), cos(rotz), 0,
 69 |                    0, 0, 1);
 70 | 
 71 |     cv::Mat R = R_z * R_y * R_x;
 72 | 
 73 |     return R;
 74 | }
 75 | 
 76 | auto Equi2Rect::reprojection(int x_img, int y_img) -> cv::Vec2d
 77 | {
 78 |     cv::Mat xyz = (cv::Mat_<double>(3, 1) << (double)x_img, (double)y_img, 1);
 79 |     cv::Mat ray3d = Rot * K.inv() * xyz / norm(xyz);
 80 | 
 81 |     // get 3d spherical coordinates
 82 |     double xp = ray3d.at<double>(0);
 83 |     double yp = ray3d.at<double>(1);
 84 |     double zp = ray3d.at<double>(2);
 85 |     // inverse formula for spherical projection, reference Szeliski book "Computer Vision: Algorithms and Applications" p439.
 86 |     double theta = atan2(yp, sqrt(xp * xp + zp * zp));
 87 |     double phi = atan2(xp, zp);
 88 | 
 89 |     // get 2D point on equirectangular map
 90 |     double x_sphere = (((phi * img_src.cols) / M_PI + img_src.cols) / 2);
 91 |     double y_sphere = (theta + M_PI / 2) * img_src.rows / M_PI;
 92 | 
 93 |     return cv::Vec2d(x_sphere, y_sphere);
 94 | }
 95 | 
 96 | auto Equi2Rect::bilinear_interpolation() -> void
 97 | {
 98 |     cv::Vec2d current_pos;
 99 |     // variables for bilinear interpolation
100 |     int top_left_x, top_left_y;
101 |     double dx, dy, wtl, wtr, wbl, wbr;
102 |     cv::Vec3b value, bgr;
103 | 
104 |     // loop over every pixel in output rectlinear image
105 |     for (int v = 0; v < viewport.height; ++v)
106 |     {
107 |         for (int u = 0; u < viewport.width; ++u)
108 |         {
109 | 
110 |             // determine corresponding position in the equirectangular panorama
111 |             current_pos = reprojection(u, v);
112 | 
113 |             // determine the nearest top left pixel for bilinear interpolation
114 |             top_left_x = static_cast<int>(current_pos[0]); // convert the subpixel value to a proper pixel value (top left pixel due to int() operator)
115 |             top_left_y = static_cast<int>(current_pos[1]);
116 | 
117 |             // if the current position exceeeds the panorama image limit -- leave pixel black and skip to next iteration
118 |             if (current_pos[0] < 0 || top_left_x > img_src.cols - 1 || current_pos[1] < 0 || top_left_y > img_src.rows - 1)
119 |             {
120 |                 continue;
121 |             }
122 | 
123 |             // initialize weights for bilinear interpolation
124 |             dx = current_pos[0] - top_left_x;
125 |             dy = current_pos[1] - top_left_y;
126 |             wtl = (1.0 - dx) * (1.0 - dy);
127 |             wtr = dx * (1.0 - dy);
128 |             wbl = (1.0 - dx) * dy;
129 |             wbr = dx * dy;
130 | 
131 |             // determine subpixel value with bilinear interpolation
132 |             bgr = wtl * img_src.at<cv::Vec3b>(top_left_y, top_left_x) + wtr * img_src.at<cv::Vec3b>(top_left_y, top_left_x + 1) +
133 |                   wbl * img_src.at<cv::Vec3b>(top_left_y + 1, top_left_x) + wbr * img_src.at<cv::Vec3b>(top_left_y + 1, top_left_x + 1);
134 | 
135 |             // paint the pixel in the output image with the calculated value
136 |             img_dst.at<cv::Vec3b>(cv::Point(u, v)) = bgr;
137 |         }
138 |     }
139 |     return;
140 | }


--------------------------------------------------------------------------------
/Equi2Rect.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef EQUI2RECT_HPP
 2 | #define EQUI2RECT_HPP
 3 | 
 4 | #include "opencv2/core/core.hpp"
 5 | #include "opencv2/highgui/highgui.hpp"
 6 | #include "opencv2/imgproc/imgproc.hpp"
 7 | 
 8 | struct ViewPortSettings
 9 | {
10 |     double pan_angle;
11 |     double tilt_angle;
12 |     double roll_angle;
13 |     int width;
14 |     int height;
15 | };
16 | 
17 | class Equi2Rect
18 | {
19 | public:
20 |     Equi2Rect();
21 |     auto save_rectlinear_image() -> void;
22 |     auto show_rectlinear_image() -> void;
23 | 
24 | private:
25 |     auto eul2rotm(double rotx, double roty, double rotz) -> cv::Mat;
26 |     auto bilinear_interpolation() -> void;
27 |     auto reprojection(int x_img, int y_img) -> cv::Vec2d;
28 | 
29 |     ViewPortSettings viewport;
30 |     int focal_length;
31 |     cv::Mat Rot;
32 |     cv::Mat K;
33 |     cv::Mat img_src;
34 |     cv::Mat img_dst;
35 |     std::string img_dst_path;
36 | };
37 | 
38 | #endif


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Ruofan
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Extract rectlinear image from panorama
 2 | 
 3 | **Disclaimer:** The code merely concentrates on how to obtain a rectlinear image (undistorted for our human eye) manually without fancy 3D rendering and texture mapping. It is not optimized to be efficient, just a small experiment project to understand the theory better.
 4 | 
 5 | 
 6 | The maths behind the reprojection is pretty staight forward [1]: The panorama is reprojected on the final image plane by determining the normalized 3D ray for each pixel in the rectlinear output image and converting it to spherical coordinates.          
 7 | With these spherical coordinates we obtain the latitude and longitude of the point on the sphere and compute the corresponding position on the equirectangular panorama to determine the source pixel intensity value.
 8 | After acquiring the correspondence between output image pixel and equirectangular source pixel,
 9 | bilinear interpolation is used to reproject the image.
10 | 
11 | The c++ code uses OpenCV (>= 3.0) to read/ output the images.        
12 | The input image should be a 2:1 360x180 degrees field of view image to work properly.
13 | 
14 | [1]: Szeliski book "Computer Vision: Algorithms and Applications" p439
15 | 
16 | 
17 | # Required dependencies
18 | - OpenCV >= 3.0 (should be compiled with GUI support, using e.g. libgtk2.0, otherwise `cv::imshow` will fail)
19 | - libyaml-cpp-dev (`sudo apt install libyaml-cpp-dev`)
20 | 
21 | # Build & use code
22 | From the project root, run:
23 | ```
24 | mkdir build
25 | cd build
26 | cmake ..
27 | make
28 | cd .. && ./build/equi2rect_example
29 | ```
30 | The output image is written to the specified path in the `config.yaml` file. A window will additionally pop up, showing the output image (might take some seconds).
31 | You can play around with the values in the config file to get different output images.
32 | 
33 | Example images:      
34 | ![original image](/images/pano.jpg)
35 | ![rectified image](/images/pano_rect.jpg "Rectified image")
36 | Rectified image with 20 degrees pan angle and 10 degrees tilt angle. 
37 | 
38 | ## Config file for output settings
39 | ```yaml
40 | camera:
41 |   focal_length: 672
42 | files:
43 |   img_dst_path: images/pano_rect.jpg # relativ path from directory where binary is executed
44 |   img_src_path: images/pano.jpg
45 | viewport:
46 |   height: 1080 # px
47 |   # angles are in radians
48 |   pan_angle: 0.349066
49 |   roll_angle: 0.0
50 |   tilt_angle: 0.174533
51 |   width: 960 # px
52 | ```
53 | 


--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
 1 | camera:
 2 |   focal_length: 672
 3 | files:
 4 |   img_dst_path: images/pano_rect.jpg
 5 |   img_src_path: images/pano.jpg
 6 | viewport:
 7 |   height: 1080
 8 |   pan_angle: 0.349066
 9 |   roll_angle: 0.0
10 |   tilt_angle: 0.174533
11 |   width: 960
12 | 


--------------------------------------------------------------------------------
/images/pano.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rfn123/equirectangular-to-rectlinear/bf946bfc1028bf4fd5778e8261e57cd21402b4ec/images/pano.jpg


--------------------------------------------------------------------------------
/images/pano_rect.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rfn123/equirectangular-to-rectlinear/bf946bfc1028bf4fd5778e8261e57cd21402b4ec/images/pano_rect.jpg


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
 1 | #include "Equi2Rect.hpp"
 2 | #include <iostream>
 3 | #include <stdio.h>
 4 | #include <time.h>
 5 | 
 6 | #define LOG(msg) std::cout << msg << std::endl
 7 | 
 8 | int main(int argc, const char **argv)
 9 | {
10 | 
11 |     clock_t tStart = clock();
12 | 
13 |     Equi2Rect equi2rect;
14 | 
15 |     equi2rect.save_rectlinear_image();
16 | 
17 |     printf("Time taken: %.2fs\n", (double)(clock() - tStart) / CLOCKS_PER_SEC);
18 | 
19 |     equi2rect.show_rectlinear_image();
20 | 
21 |     return 0;
22 | }


--------------------------------------------------------------------------------