├── rustfmt.toml ├── akaze ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── LICENSE ├── algorithm.md ├── examples │ └── akaze.rs ├── README.md ├── benches │ └── criterion.rs ├── Cargo.toml ├── src │ ├── contrast_factor.rs │ ├── derivatives.rs │ ├── nonlinear_diffusion.rs │ ├── detector_response.rs │ ├── fed_tau.rs │ ├── evolution.rs │ └── descriptors.rs └── tests │ └── estimate_pose.rs ├── cv-core ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── src │ ├── matches.rs │ ├── camera.rs │ ├── keypoint.rs │ ├── triangulation.rs │ ├── lib.rs │ ├── point.rs │ └── so3.rs ├── Cargo.toml ├── LICENSE ├── README.md └── notes │ └── derivation_of_triangulate_bearing_reproject.md ├── cv-geom ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── README.md ├── Cargo.toml ├── src │ └── lib.rs └── LICENSE ├── cv-pinhole ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── README.md ├── Cargo.toml └── LICENSE ├── eight-point ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── README.md ├── Cargo.toml ├── LICENSE ├── tests │ └── random.rs └── src │ └── lib.rs ├── tutorial-code ├── chapter2-first-program │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── chapter3-akaze-feature-extraction │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── chapter4-feature-matching │ ├── Cargo.toml │ └── src │ │ └── main.rs └── chapter5-geometric-verification │ ├── Cargo.toml │ └── src │ └── main.rs ├── lambda-twist ├── ensure_no_std │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── katex-header.html ├── Cargo.toml ├── README.md └── tests │ └── consensus.rs ├── res ├── 0000000000.png ├── 0000000014.png ├── source.txt └── calib_cam_to_cam.txt ├── kpdraw ├── README.md ├── Cargo.toml ├── src │ ├── lib.rs │ └── main.rs └── LICENSE ├── cv-optimize ├── src │ ├── lib.rs │ ├── single_view_optimizer.rs │ └── three_view_optimizer.rs ├── Cargo.toml ├── README.md └── LICENSE ├── .gitignore ├── tutorial ├── book.toml └── src │ ├── SUMMARY.md │ ├── chapter1-introduction.md │ ├── chapter2-first-program.md │ └── chapter3-akaze-feature-extraction.md ├── nister-stewenius ├── Cargo.toml ├── LICENSE ├── README.md └── tests │ └── manual.rs ├── vslam-sandbox ├── README.md ├── Cargo.toml ├── LICENSE └── src │ └── main.rs ├── Cargo.toml ├── .github └── workflows │ ├── tests.yml │ └── lints.yml ├── cv ├── LICENSE ├── README.md ├── Cargo.toml └── src │ └── lib.rs └── cv-sfm ├── Cargo.toml └── src ├── bicubic.rs └── export.rs /rustfmt.toml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /akaze/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /cv-core/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /cv-geom/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /cv-pinhole/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /eight-point/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /tutorial-code/chapter2-first-program/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /lambda-twist/ensure_no_std/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /tutorial-code/chapter3-akaze-feature-extraction/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /res/0000000000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rust-cv/cv/HEAD/res/0000000000.png -------------------------------------------------------------------------------- /res/0000000014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rust-cv/cv/HEAD/res/0000000014.png -------------------------------------------------------------------------------- /kpdraw/README.md: -------------------------------------------------------------------------------- 1 | # kpdraw 2 | 3 | A library and binary to draw keypoints in images using different keypoint detection algorithms 4 | -------------------------------------------------------------------------------- /cv-optimize/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod single_view_optimizer; 2 | mod three_view_optimizer; 3 | 4 | pub use single_view_optimizer::*; 5 | pub use three_view_optimizer::*; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # VS Code 2 | .vscode 3 | 4 | # Cargo 5 | /target 6 | **/*.rs.bk 7 | Cargo.lock 8 | 9 | # mdbook 10 | /tutorial/book/ 11 | 12 | # MacOS 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /tutorial/book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | authors = ["codec-abc", "Geordon Worley "] 3 | language = "en" 4 | multilingual = false 5 | src = "src" 6 | title = "Rust-CV Tutorials" 7 | -------------------------------------------------------------------------------- /tutorial/src/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | - [1. Introduction](./chapter1-introduction.md) 4 | - [2. Setup and First Program](./chapter2-first-program.md) 5 | - [3. Akaze feature extraction](./chapter3-akaze-feature-extraction.md) 6 | - [4. Feature matching](./chapter4-feature-matching.md) 7 | -------------------------------------------------------------------------------- /akaze/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | akaze = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /cv-core/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv-core = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /cv-geom/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv-geom = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /cv-pinhole/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv-pinhole = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /eight-point/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | eight-point = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /lambda-twist/ensure_no_std/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ensure_no_std" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | lambda-twist = { path = ".." } 9 | 10 | [profile.dev] 11 | panic = "abort" 12 | 13 | [profile.release] 14 | panic = "abort" 15 | -------------------------------------------------------------------------------- /tutorial-code/chapter2-first-program/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chapter2-first-program" 3 | version = "0.1.0" 4 | authors = ["codec-abc "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv = { version = "0.6.0", path = "../../cv" } 9 | open = "2.0.2" 10 | rand = "0.8.4" 11 | tempfile = "3.3.0" 12 | -------------------------------------------------------------------------------- /kpdraw/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kpdraw" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | akaze = { version = "0.7.0", path = "../akaze" } 9 | structopt = "0.3.26" 10 | image = { version = "0.25", features = ["png", "jpeg"] } 11 | imageproc = "0.25" 12 | -------------------------------------------------------------------------------- /akaze/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /cv-core/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /cv-geom/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /cv-pinhole/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /eight-point/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /lambda-twist/ensure_no_std/src/main.rs: -------------------------------------------------------------------------------- 1 | // ensure_no_std/src/main.rs 2 | #![no_std] 3 | #![no_main] 4 | 5 | use core::panic::PanicInfo; 6 | 7 | /// This function is called on panic. 8 | #[panic_handler] 9 | fn panic(_info: &PanicInfo) -> ! { 10 | loop {} 11 | } 12 | 13 | #[no_mangle] 14 | pub extern "C" fn _start() -> ! { 15 | loop {} 16 | } 17 | -------------------------------------------------------------------------------- /tutorial-code/chapter4-feature-matching/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chapter4-feature-matching" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | cv = { version = "0.6.0", path = "../../cv" } 8 | image = "0.25" 9 | imageproc = "0.25" 10 | open = "2.0.2" 11 | tempfile = "3.3.0" 12 | itertools = "0.10.3" 13 | palette = "0.6.0" 14 | -------------------------------------------------------------------------------- /res/source.txt: -------------------------------------------------------------------------------- 1 | This data is from collection 2011_09_26_drive_0035 of the Kitti dataset. It was accessed at: 2 | 3 | https://s3.eu-central-1.amazonaws.com/avg-kitti/raw_data/2011_09_26_drive_0035/2011_09_26_drive_0035_extract.zip 4 | 5 | You can also find the original calibration data source here: 6 | 7 | https://s3.eu-central-1.amazonaws.com/avg-kitti/raw_data/2011_09_26_calib.zip -------------------------------------------------------------------------------- /tutorial-code/chapter3-akaze-feature-extraction/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chapter3-akaze-feature-extraction" 3 | version = "0.1.0" 4 | authors = ["codec-abc "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv = { version = "0.6.0", path = "../../cv" } 9 | image = "0.25" 10 | imageproc = "0.25" 11 | open = "2.0.2" 12 | tempfile = "3.3.0" 13 | -------------------------------------------------------------------------------- /tutorial-code/chapter5-geometric-verification/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chapter5-geometric-verification" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | cv = { version = "0.6.0", path = "../../cv" } 8 | image = "0.25" 9 | imageproc = "0.25" 10 | open = "2.0.2" 11 | tempfile = "3.3.0" 12 | itertools = "0.10.3" 13 | palette = "0.6.0" 14 | rand = "0.8.4" 15 | rand_xoshiro = "0.6.0" 16 | -------------------------------------------------------------------------------- /cv-optimize/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv-optimize" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | cv-core = { version = "0.15.0", path = "../cv-core" } 9 | cv-geom = { version = "0.7.0", path = "../cv-geom" } 10 | itertools = "0.10.3" 11 | log = { version = "0.4.14", default-features = false } 12 | float-ord = { version = "0.3.2", default-features = false } 13 | -------------------------------------------------------------------------------- /cv-core/src/matches.rs: -------------------------------------------------------------------------------- 1 | use crate::WorldPoint; 2 | use nalgebra::UnitVector3; 3 | 4 | /// Two keypoint bearings matched together from two separate images 5 | #[derive(Debug, Clone, Copy, PartialEq)] 6 | pub struct FeatureMatch(pub UnitVector3, pub UnitVector3); 7 | 8 | /// A keypoint bearing matched to a [`WorldPoint`]. 9 | #[derive(Debug, Clone, Copy, PartialEq)] 10 | pub struct FeatureWorldMatch(pub UnitVector3, pub WorldPoint); 11 | -------------------------------------------------------------------------------- /nister-stewenius/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nister-stewenius" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | license = "MIT" 7 | 8 | [dependencies] 9 | arrayvec = { version = "0.7.2", default-features = false } 10 | cv-core = { version = "0.15.0", path = "../cv-core" } 11 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 12 | float-ord = "0.3.2" 13 | sample-consensus = "1.0.2" 14 | 15 | [dev-dependencies] 16 | itertools = "0.10.3" 17 | nalgebra = "0.30.1" 18 | -------------------------------------------------------------------------------- /kpdraw/src/lib.rs: -------------------------------------------------------------------------------- 1 | use akaze::{Akaze, KeyPoint}; 2 | use image::{DynamicImage, Rgba}; 3 | use imageproc::drawing; 4 | 5 | pub fn render_akaze_keypoints(image: &DynamicImage, threshold: f64) -> DynamicImage { 6 | let akaze = Akaze::new(threshold); 7 | let (kps, _) = akaze.extract(image); 8 | let mut image = drawing::Blend(image.to_rgba8()); 9 | for KeyPoint { point: (x, y), .. } in kps { 10 | drawing::draw_cross_mut(&mut image, Rgba([0, 255, 255, 128]), x as i32, y as i32); 11 | } 12 | DynamicImage::ImageRgba8(image.0) 13 | } 14 | -------------------------------------------------------------------------------- /cv-geom/README.md: -------------------------------------------------------------------------------- 1 | # cv-geom 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/cv-geom.svg 6 | [cl]: https://crates.io/crates/cv-geom/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/cv-geom/badge.svg 11 | [dl]: https://docs.rs/cv-geom/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | Collection of computational geometry algorithms for Rust CV 17 | -------------------------------------------------------------------------------- /cv-optimize/README.md: -------------------------------------------------------------------------------- 1 | # cv-optimize 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/cv-optimize.svg 6 | [cl]: https://crates.io/crates/cv-optimize/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/cv-optimize/badge.svg 11 | [dl]: https://docs.rs/cv-optimize/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | Provides optimizers for common computer vision problems 17 | -------------------------------------------------------------------------------- /cv-geom/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv-geom" 3 | version = "0.7.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "Contains computational geometry algorithms used in computer vision" 7 | documentation = "https://docs.rs/cv-geom/" 8 | repository = "https://github.com/rust-cv/cv" 9 | keywords = ["computer", "vision", "computational", "geometry", "triangulation"] 10 | categories = ["computer-vision", "no-std", "science::robotics"] 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [dependencies] 15 | cv-core = { version = "0.15.0", path = "../cv-core" } 16 | float-ord = "0.3.2" 17 | -------------------------------------------------------------------------------- /eight-point/README.md: -------------------------------------------------------------------------------- 1 | # eight-point 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/eight-point.svg 6 | [cl]: https://crates.io/crates/eight-point/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/eight-point/badge.svg 11 | [dl]: https://docs.rs/eight-point/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | Implements the [eight-point algorithm](https://en.wikipedia.org/wiki/Eight-point_algorithm) by Richard Hartley and Andrew Zisserman for estimating the essential matrix from keypoint correspondences. 17 | -------------------------------------------------------------------------------- /vslam-sandbox/README.md: -------------------------------------------------------------------------------- 1 | # vslam-sandbox 2 | 3 | A sandbox for integrating upstream vslam algorithms 4 | 5 | ## Goal 6 | 7 | This sandbox should allow using and benchmarking the available vSLAM algorithms using the `vslam` crate as an abstraction to allow swapping out different components. 8 | 9 | ## Building 10 | 11 | Please prepend `RUSTFLAGS="-C target-cpu=native"` to your cargo commands to run this with 12 | native optimizations. Rust can perform some autovectorization via LLVM, but it needs to be 13 | told that its okay that it only runs on your system. There are also some dependencies that 14 | explicitly use the best available SIMD instructions when they are available, which they 15 | aren't by default. The above environment variable will fix that and allow it to use AVX-512 16 | or AVX2 depending on what is available. 17 | -------------------------------------------------------------------------------- /eight-point/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "eight-point" 3 | version = "0.8.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "Eight-point algorithm for essential matrix estimation" 7 | documentation = "https://docs.rs/eight-point/" 8 | repository = "https://github.com/rust-cv/cv" 9 | keywords = ["hartly", "zisserman", "photogrammetry", "eight", "point"] 10 | categories = ["algorithms", "computer-vision", "no-std", "science", "science::robotics"] 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [dependencies] 15 | cv-core = { version = "0.15.0", path = "../cv-core" } 16 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 17 | float-ord = "0.3.2" 18 | derive_more = "0.99.17" 19 | num-traits = { version = "0.2.14", default-features = false } 20 | arrayvec = { version = "0.7.2", default-features = false } 21 | 22 | [dev-dependencies] 23 | nalgebra = { version = "0.30.1", features = ["rand"] } 24 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "cv", 5 | "cv-core", 6 | "cv-geom", 7 | "cv-pinhole", 8 | "cv-optimize", 9 | "cv-sfm", 10 | "akaze", 11 | "eight-point", 12 | "lambda-twist", 13 | "nister-stewenius", 14 | "vslam-sandbox", 15 | "kpdraw", 16 | "tutorial-code/chapter2-first-program", 17 | "tutorial-code/chapter3-akaze-feature-extraction", 18 | "tutorial-code/chapter4-feature-matching", 19 | "tutorial-code/chapter5-geometric-verification", 20 | ] 21 | 22 | [profile.dev] 23 | # The tests take a very long time without optimization. 24 | opt-level = 1 25 | # This is needed to reduce memory usage during compilation, or CI will fail 26 | codegen-units = 1 27 | 28 | [profile.bench] 29 | # Necessary to generate flamegraphs 30 | debug = true 31 | codegen-units = 1 32 | lto = "fat" 33 | 34 | [profile.release] 35 | # Necessary to generate flamegraphs 36 | debug = true 37 | codegen-units = 1 38 | lto = "fat" 39 | -------------------------------------------------------------------------------- /cv-pinhole/README.md: -------------------------------------------------------------------------------- 1 | # cv-pinhole 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/cv-pinhole.svg 6 | [cl]: https://crates.io/crates/cv-pinhole/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/cv-pinhole/badge.svg 11 | [dl]: https://docs.rs/cv-pinhole/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | Pinhole camera model for Rust CV 17 | 18 | This crate seamlessly plugs into `cv-core` and provides pinhole camera models with and without distortion correction. 19 | It can be used to convert image coordinates into real 3d direction vectors (called bearings) pointing towards where 20 | the light came from that hit that pixel. It can also be used to convert backwards from the 3d back to the 2d 21 | using the `uncalibrate` method from the `cv_core::CameraModel` trait. 22 | -------------------------------------------------------------------------------- /cv-geom/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate contains computational geometry algorithms for [Rust CV](https://github.com/rust-cv/). 2 | //! 3 | //! ## Triangulation 4 | //! 5 | //! In this problem we know the relative pose of cameras and the bearing of the same feature 6 | //! observed in each camera frame. We want to find the point of intersection from all cameras. 7 | //! 8 | //! - `p` the point we are trying to triangulate 9 | //! - `a` the normalized keypoint on camera A 10 | //! - `b` the normalized keypoint on camera B 11 | //! - `O` the optical center of a camera 12 | //! - `@` the virtual image plane 13 | //! 14 | //! ```text 15 | //! @ 16 | //! @ 17 | //! p--------b--------O 18 | //! / @ 19 | //! / @ 20 | //! / @ 21 | //! / @ 22 | //! @@@@@@@a@@@@@ 23 | //! / 24 | //! / 25 | //! / 26 | //! O 27 | //! ``` 28 | 29 | #![no_std] 30 | 31 | pub mod epipolar; 32 | pub mod triangulation; 33 | -------------------------------------------------------------------------------- /cv-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv-core" 3 | version = "0.15.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "Contains core primitives used in computer vision applications" 7 | documentation = "https://docs.rs/cv-core/" 8 | repository = "https://github.com/rust-cv/cv" 9 | keywords = ["computer", "vision", "core", "cv", "photogrammetry"] 10 | categories = ["algorithms", "computer-vision", "no-std", "science", "science::robotics"] 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [features] 15 | serde-serialize = ["serde", "nalgebra/serde-serialize"] 16 | 17 | [dependencies] 18 | # TODO: Fix this once alloc feature is working again. 19 | nalgebra = { version = "0.30.1", default-features = false, features = ["std"] } 20 | derive_more = "0.99.17" 21 | sample-consensus = "1.0.2" 22 | num-traits = { version = "0.2.14", default-features = false } 23 | serde = { version = "1.0.136", default-features = false, features = ["derive"], optional = true } 24 | 25 | [package.metadata.docs.rs] 26 | all-features = true 27 | -------------------------------------------------------------------------------- /vslam-sandbox/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "vslam-sandbox" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [features] 8 | unstable-512-bit-simd = ["bitarray/unstable-512-bit-simd"] 9 | 10 | [dependencies] 11 | cv-core = { version = "0.15.0", path = "../cv-core" } 12 | cv-sfm = { version = "0.1.0", path = "../cv-sfm", features = [ 13 | "serde-serialize", 14 | ] } 15 | cv-geom = { version = "0.7.0", path = "../cv-geom" } 16 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 17 | lambda-twist = { version = "0.7.0", path = "../lambda-twist" } 18 | eight-point = { version = "0.8.0", path = "../eight-point" } 19 | arrsac = "0.10.0" 20 | structopt = "0.3.26" 21 | serde = { version = "1.0.136", features = ["derive"] } 22 | image = "0.25" 23 | rand = "0.8.4" 24 | rand_xoshiro = "0.6.0" 25 | log = "0.4.14" 26 | pretty_env_logger = "0.4.0" 27 | bincode = "1.3.3" 28 | serde_json = "1.0.78" 29 | bitarray = { version = "0.9.0", default-features = false } 30 | slotmap = { version = "1.0.6", default-features = false } 31 | -------------------------------------------------------------------------------- /lambda-twist/katex-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | pull_request: 6 | 7 | name: tests 8 | 9 | jobs: 10 | tests: 11 | name: tests 12 | runs-on: ubuntu-latest 13 | env: 14 | CARGO_BACKTRACE: 1 15 | steps: 16 | - name: Install system dependencies 17 | run: | 18 | sudo apt-get -y install libxkbcommon-dev 19 | 20 | - name: Checkout sources 21 | uses: actions/checkout@v2 22 | 23 | - name: Install beta toolchain 24 | uses: actions-rs/toolchain@v1 25 | with: 26 | profile: minimal 27 | toolchain: beta 28 | override: true 29 | 30 | - uses: Swatinem/rust-cache@v1 31 | with: 32 | cache-on-failure: true 33 | 34 | - name: Run cargo test 35 | uses: actions-rs/cargo@v1 36 | with: 37 | command: test 38 | args: --features cv/serde-serialize 39 | 40 | - name: Run cargo test for Akaze with rayon feature 41 | uses: actions-rs/cargo@v1 42 | with: 43 | command: test 44 | args: --features akaze/rayon -------------------------------------------------------------------------------- /cv-pinhole/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv-pinhole" 3 | version = "0.6.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "Pinhole camera model for comptuer vision" 7 | documentation = "https://docs.rs/cv-pinhole/" 8 | repository = "https://github.com/rust-cv/cv" 9 | keywords = ["computer", "vision", "pinhole", "camera", "calibration"] 10 | categories = ["algorithms", "computer-vision", "no-std", "science::robotics"] 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [features] 15 | default = [] 16 | alloc = [] 17 | serde-serialize = ["serde", "nalgebra/serde-serialize"] 18 | 19 | [dependencies] 20 | cv-core = { version = "0.15.0", path = "../cv-core" } 21 | derive_more = "0.99.17" 22 | num-traits = { version = "0.2.14", default-features = false } 23 | float-ord = "0.3.2" 24 | serde = { version = "1.0.136", features = ["derive"], default-features = false, optional = true } 25 | nalgebra = { version = "0.30.1", default-features = false } 26 | 27 | [dev-dependencies] 28 | cv-geom = { version = "0.7.0", path = "../cv-geom" } 29 | 30 | [package.metadata.docs.rs] 31 | all-features = true 32 | -------------------------------------------------------------------------------- /akaze/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 rust-cv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cv-core/src/camera.rs: -------------------------------------------------------------------------------- 1 | use crate::{ImagePoint, KeyPoint}; 2 | use nalgebra::UnitVector3; 3 | 4 | /// Allows conversion between the point on an image and the internal projection 5 | /// which can describe the bearing of the projection out of the camera. 6 | pub trait CameraModel { 7 | /// Extracts a bearing from a pixel location in an image. 8 | /// 9 | /// The bearings X axis points right, Y axis points down, and Z axis points forwards. 10 | /// The image point uses the same coordiate frame. Its Y is down and its X is right. 11 | fn calibrate

(&self, point: P) -> UnitVector3 12 | where 13 | P: ImagePoint; 14 | 15 | /// Extracts the pixel location in the image from the bearing. 16 | /// 17 | /// The bearings X axis points right, Y axis points down, and Z axis points forwards. 18 | /// The image point uses the same coordiate frame. Its Y is down and its X is right. 19 | /// 20 | /// Since this might not be possible (if bearing is behind camera for pinhole camera), 21 | /// this operation is fallible. 22 | fn uncalibrate(&self, bearing: UnitVector3) -> Option; 23 | } 24 | -------------------------------------------------------------------------------- /akaze/algorithm.md: -------------------------------------------------------------------------------- 1 | # AKAZE explained 2 | 3 | At a high level, AKAZE is an algorithm that takes in an image, determines where features 4 | are at different scales in the image, then extracts gradient information for matching at that point. 5 | 6 | The first thing AKAZE does is create a scale-space pyramid. This pyramid is a series of 7 | images containing the scale estimation of the image. The higher you go in the pyramid, the scale at which the edges are evaluated changes. 8 | 9 | AKAZE does this by setting up a series of three octaves. Each octave is a 2x scaling in the image dimensions, so a 512x512 scale space in octave 0 would become a 256x256 scale space in octave 1. Typically there are three octaves in total: 0, 1, and 2. 10 | 11 | AKAZE then further divides these octaves up into sublevels. The typical amount of sublevels to use is four: 0, 1, 2, and 3. 12 | 13 | In essence, each `octave,sublevel` pair forms a layer of the pyramid. In the typical case, it looks like this: 14 | 15 | 0. `0,0` 16 | 1. `0,1` 17 | 2. `0,2` 18 | 3. `0,3` 19 | 4. `1,0` 20 | 5. `1,1` 21 | 6. `1,2` 22 | 7. `1,3` 23 | 8. `2,0` 24 | 9. `2,1` 25 | 10. `2,2` 26 | 11. `2,3` 27 | -------------------------------------------------------------------------------- /cv-core/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 rust-cv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cv/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rust Computer Vision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /eight-point/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 rust-cv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /vslam-sandbox/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 rust-cv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cv-geom/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rust Computer Vision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /kpdraw/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rust Computer Vision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lambda-twist/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lambda-twist" 3 | version = "0.7.0" 4 | authors = ["Matthieu Pizenberg ", "Geordon Worley "] 5 | edition = "2021" 6 | description = "p3p pose estimation given world points and feature bearings" 7 | documentation = "https://docs.rs/lambda-twist/" 8 | repository = "https://github.com/rust-cv/cv" 9 | readme = "README.md" 10 | keywords = ["p3p", "pose", "vision", "nordberg"] 11 | categories = ["computer-vision", "algorithms", "science::robotics", "no-std"] 12 | license = "MPL-2.0" 13 | 14 | [package.metadata.docs.rs] 15 | rustdoc-args = [ "--html-in-header", "katex-header.html" ] 16 | 17 | [dependencies] 18 | cv-core = { version = "0.15.0", path = "../cv-core" } 19 | num-traits = { version = "0.2.14", default-features = false } 20 | arrayvec = { version = "0.7.2", default-features = false } 21 | 22 | [dev-dependencies] 23 | approx = "0.5.1" 24 | arrsac = "0.10.0" 25 | rand = { version = "0.8.4", features = ["small_rng"] } 26 | quickcheck = "1.0.3" 27 | quickcheck_macros = "1.0.0" 28 | itertools = "0.10.3" 29 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 30 | -------------------------------------------------------------------------------- /nister-stewenius/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 rust-cv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cv-optimize/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rust Computer Vision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cv-pinhole/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rust Computer Vision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /nister-stewenius/README.md: -------------------------------------------------------------------------------- 1 | # nister-stewenius 2 | 3 | Essential matrix estimation from 5 normalized image coordinate correspondences from the paper "Recent developments on direct relative orientation" 4 | 5 | This crate implements the `Estimator` trait from the `sample-consensus` crate. This allows integration with sample consensus algorithms built on that crate. The model returned by this crate is an essential matrix which is estimated from 5 points. On each estimation, up to `10` solutions may be returned. It is recommended to use the `arrsac` crate on your data with the estimator in this crate to get the best essential matrix for your data. 6 | 7 | ## Testing 8 | 9 | Note that when `cargo test` is ran, since this crate builds by default with `no_std`, it spits out some errors when trying to run the doc tests: 10 | ``` 11 | error: no global memory allocator found but one is required; link to std or add `#[global_allocator]` to a static item that implements the GlobalAlloc trait. 12 | 13 | error: aborting due to previous error 14 | ``` 15 | 16 | This error does not cause the unit tests to fail, nor does cargo return an error code. It can be annoying, but simply scroll up to see the unit test results. This happens due to https://github.com/rust-lang/rust/issues/54010. 17 | -------------------------------------------------------------------------------- /akaze/examples/akaze.rs: -------------------------------------------------------------------------------- 1 | use std::{fs, path::Path}; 2 | 3 | use akaze::Akaze; 4 | 5 | fn replace_ext(filename: &str, new: &str) -> String { 6 | let stemmed = Path::new(filename).file_stem().unwrap().to_str().unwrap(); 7 | format!("{stemmed}{new}") 8 | } 9 | 10 | fn main() { 11 | let args: Vec<_> = std::env::args().collect(); 12 | for path in &args[1..] { 13 | let kps = Akaze::default().extract_path(path).unwrap(); 14 | let mut kp_file = fs::File::create(replace_ext(path, "_kps.csv")).unwrap(); 15 | let mut desc_file = fs::File::create(replace_ext(path, "_descs.txt")).unwrap(); 16 | for (kp, descriptor) in kps.0.iter().zip(kps.1.iter()) { 17 | std::io::Write::write_all( 18 | &mut kp_file, 19 | format!( 20 | "{}, {}, {}, {}, {}, {}\n", 21 | kp.point.0, kp.point.1, kp.angle, kp.size, kp.octave, kp.class_id 22 | ) 23 | .as_bytes(), 24 | ) 25 | .unwrap(); 26 | std::io::Write::write_all( 27 | &mut desc_file, 28 | format!("{}\n", descriptor.map(|x| format!("{x:08b}")).join("_")).as_bytes(), 29 | ) 30 | .unwrap(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /cv/README.md: -------------------------------------------------------------------------------- 1 | # cv 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/cv.svg 6 | [cl]: https://crates.io/crates/cv/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/cv/badge.svg 11 | [dl]: https://docs.rs/cv/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | Batteries-included pure-Rust computer vision crate 17 | 18 | Unlike other crates in the rust-cv ecosystem, this crate enables all features by-default as part of its batteries-included promise. The features can be turned off if desired by setting `default-features = false` for the package in Cargo.toml. However, it is recommended that if you only want specific rust-cv components that you add those components individually to better control dependency bloat and build times. This crate is useful for experimentation, but it is recommended that before you publish a crate or deploy a binary that you use the component crates and not `cv`. The `cv` crate itself provides no functionality, but only provides a useful organization of computer vision components. 19 | 20 | This crate is `no_std` capable, but you must turn off some of the default enabled features to achieve this. 21 | -------------------------------------------------------------------------------- /cv-core/src/keypoint.rs: -------------------------------------------------------------------------------- 1 | use derive_more::{AsMut, AsRef, Deref, DerefMut, From, Into}; 2 | use nalgebra::Point2; 3 | 4 | #[cfg(feature = "serde-serialize")] 5 | use serde::{Deserialize, Serialize}; 6 | 7 | /// Allows the retrieval of the point on the image the feature came from. 8 | /// 9 | /// The origin for an image point is in the top left of the image. Positive X axis points right 10 | /// and positive Y axis points down. 11 | pub trait ImagePoint { 12 | /// Retrieves the point on the image 13 | fn image_point(&self) -> Point2; 14 | } 15 | 16 | /// A point on an image frame. This type should be used when 17 | /// the point location is on the image frame in pixel coordinates. 18 | /// This means the keypoint is neither undistorted nor normalized. 19 | /// 20 | /// For calibrated coordinates, you need to use an appropriate camera model crate (like `cv-pinhole`). 21 | /// These crates convert image coordinates into bearings. For more information, see the trait definition 22 | /// [`crate::CameraModel`]. 23 | #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, AsMut, AsRef, Deref, DerefMut, From, Into)] 24 | #[cfg_attr(feature = "serde-serialize", derive(Serialize, Deserialize))] 25 | pub struct KeyPoint(pub Point2); 26 | 27 | impl ImagePoint for KeyPoint { 28 | fn image_point(&self) -> Point2 { 29 | self.0 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /.github/workflows/lints.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | pull_request: 6 | 7 | name: lints 8 | 9 | jobs: 10 | rustfmt: 11 | name: rustfmt 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout sources 15 | uses: actions/checkout@v2 16 | 17 | - name: Install beta toolchain 18 | uses: actions-rs/toolchain@v1 19 | with: 20 | profile: minimal 21 | toolchain: beta 22 | override: true 23 | components: rustfmt 24 | 25 | - uses: Swatinem/rust-cache@v1 26 | with: 27 | cache-on-failure: true 28 | 29 | - name: Run cargo fmt 30 | uses: actions-rs/cargo@v1 31 | with: 32 | command: fmt 33 | args: --all -- --check 34 | clippy: 35 | name: clippy 36 | runs-on: ubuntu-latest 37 | steps: 38 | - name: Checkout sources 39 | uses: actions/checkout@v2 40 | 41 | - name: Install beta toolchain 42 | uses: actions-rs/toolchain@v1 43 | with: 44 | profile: minimal 45 | toolchain: beta 46 | override: true 47 | components: clippy 48 | 49 | - uses: Swatinem/rust-cache@v1 50 | with: 51 | cache-on-failure: true 52 | 53 | - name: Run cargo clippy 54 | uses: actions-rs/cargo@v1 55 | with: 56 | command: clippy 57 | -------------------------------------------------------------------------------- /cv-core/README.md: -------------------------------------------------------------------------------- 1 | # cv-core 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/cv-core.svg 6 | [cl]: https://crates.io/crates/cv-core/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/cv-core/badge.svg 11 | [dl]: https://docs.rs/cv-core/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | This library provides common abstractions and types for computer vision (CV) in Rust. 17 | All the crates in the rust-cv ecosystem that have or depend on CV types depend on this crate. 18 | This includes things like camera model traits, bearings, poses, keypoints, etc. The crate is designed to 19 | be very small so that it adds negligable build time. It pulls in some dependencies 20 | that will probably be brought in by writing computer vision code normally. 21 | The core concept is that all CV crates can work together with each other by using the 22 | abstractions and types specified in this crate. 23 | 24 | The crate is designed to work with `#![no_std]`, even without an allocator. `libm` is used 25 | for all math algorithms that aren't present in `std`. Any code that doesn't need to be shared 26 | across all CV crates should not belong in this repository. If there is a good reason to put 27 | code that some crates may need into `cv-core`, it should be gated behind a feature. 28 | -------------------------------------------------------------------------------- /akaze/README.md: -------------------------------------------------------------------------------- 1 | # akaze 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] ![MIT/Apache][li] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/akaze.svg 6 | [cl]: https://crates.io/crates/akaze/ 7 | 8 | [li]: https://img.shields.io/badge/License-MIT-yellow.svg 9 | 10 | [di]: https://docs.rs/akaze/badge.svg 11 | [dl]: https://docs.rs/akaze/ 12 | 13 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 14 | [dcl]: https://discord.gg/d32jaam 15 | 16 | AKAZE feature extraction algorithm for computer vision 17 | 18 | Implementation of AKAZE based on the one by indianajohn. He gave me permission to copy this here and work from that, as his job conflicts with maintainership. The crate is greatly changed from the original. 19 | 20 | See `tests/estimate_pose.rs` for a demonstration on how to use this crate. 21 | 22 | This crate adds several optimizations (using ndarray) to the original implementation and integrates directly into the rust-cv ecosystem for ease-of-use. This crate does not currently use threading to speed anything up, but it might be added as a Cargo feature in the future. 23 | 24 | The original implementation can be found here: 25 | 26 | The previous rust implementation can be found here: 27 | 28 | The main site for the algorithm is normally [here](http://www.robesafe.com/personal/pablo.alcantarilla/kaze.html), but it was down, so I found another link to the paper: 29 | -------------------------------------------------------------------------------- /kpdraw/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | io::{Cursor, Write}, 3 | path::PathBuf, 4 | }; 5 | use structopt::StructOpt; 6 | 7 | #[derive(Debug, StructOpt)] 8 | #[structopt( 9 | name = "kpdraw", 10 | about = "A tool to show keypoints from different keypoint detectors" 11 | )] 12 | struct Opt { 13 | /// The akaze threshold to use. 14 | /// 15 | /// 0.01 will be very sparse and 0.0001 will be very dense. 16 | #[structopt(short, long, default_value = "0.001")] 17 | threshold: f64, 18 | /// The output path to write to (autodetects image type from extension). 19 | /// 20 | /// If this is not provided, then the output goes to stdout as a PNG. 21 | #[structopt(short, long, parse(from_os_str))] 22 | output: Option, 23 | /// The image file to show keypoints on. 24 | #[structopt(parse(from_os_str))] 25 | input: PathBuf, 26 | } 27 | 28 | fn main() { 29 | let opt = Opt::from_args(); 30 | let image = image::open(opt.input).expect("failed to open image file"); 31 | let image = kpdraw::render_akaze_keypoints(&image, opt.threshold); 32 | let stdout = std::io::stdout(); 33 | if let Some(path) = opt.output { 34 | image.save(path).expect("failed to write image to stdout"); 35 | } else { 36 | let mut output = Cursor::new(Vec::new()); 37 | image 38 | .write_to(&mut output, image::ImageFormat::Png) 39 | .expect("failed to write image to stdout"); 40 | stdout 41 | .lock() 42 | .write_all(&output.into_inner()) 43 | .expect("failed to write image to stdout"); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /cv-sfm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv-sfm" 3 | version = "0.1.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | 7 | [features] 8 | serde-serialize = [ 9 | "serde", 10 | "cv-core/serde-serialize", 11 | "cv-pinhole/serde-serialize", 12 | "bitarray/serde", 13 | "slotmap/serde", 14 | "hgg/serde", 15 | "hamming-lsh/serde", 16 | "akaze/serde", 17 | ] 18 | 19 | [dependencies] 20 | cv-core = { version = "0.15.0", path = "../cv-core" } 21 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 22 | cv-geom = { version = "0.7.0", path = "../cv-geom" } 23 | eight-point = { version = "0.8.0", path = "../eight-point" } 24 | lambda-twist = { version = "0.7.0", path = "../lambda-twist" } 25 | cv-optimize = { version = "0.1.0", path = "../cv-optimize" } 26 | akaze = { version = "0.7.0", path = "../akaze" } 27 | space = { version = "0.17.0", default-features = false } 28 | maplit = { version = "1.0.2", default-features = false } 29 | log = { version = "0.4.14", default-features = false } 30 | itertools = { version = "0.10.3", default-features = false } 31 | image = { version = "0.25", default-features = false } 32 | ply-rs = { version = "0.1.3", default-features = false } 33 | imageproc = "0.25" 34 | conv = { version = "0.3.3", default-features = false } 35 | bitarray = { version = "0.9.0", default-features = false, features = ["space"] } 36 | serde = { version = "1.0.136", features = ["derive"], optional = true } 37 | slotmap = "1.0.6" 38 | hgg = { version = "0.4.1", default-features = false } 39 | rand = { version = "0.8.4", default-features = false } 40 | hamming-lsh = "0.3.2" 41 | float-ord = { version = "0.3.2", default-features = false } 42 | average = "0.13.1" 43 | arrayvec = { version = "0.7.2", default-features = false } 44 | -------------------------------------------------------------------------------- /lambda-twist/README.md: -------------------------------------------------------------------------------- 1 | # lambda-twist 2 | 3 | [![Discord][dci]][dcl] [![Crates.io][ci]][cl] [![MPL 2.0][li]][lil] [![docs.rs][di]][dl] 4 | 5 | [ci]: https://img.shields.io/crates/v/lambda-twist.svg 6 | [cl]: https://crates.io/crates/lambda-twist/ 7 | 8 | [li]: https://img.shields.io/badge/License-MPL%202.0-brightgreen.svg 9 | [lil]: https://opensource.org/licenses/MPL-2.0 10 | 11 | [di]: https://docs.rs/lambda-twist/badge.svg 12 | [dl]: https://docs.rs/lambda-twist/ 13 | 14 | [dci]: https://img.shields.io/discord/550706294311485440.svg?logo=discord&colorB=7289DA 15 | [dcl]: https://discord.gg/d32jaam 16 | 17 | Relative camera pose from three 3d to 2d correspondences 18 | 19 | To see an example of usage, see `tests/consensus.rs`. 20 | 21 | This was derived from . It was rewritten to utilize Rust CV abstractions. 22 | 23 | Implementation based on 24 | "Lambda Twist: An Accurate Fast Robust Perspective Three Point (P3P) Solver" 25 | Persson, M. and Nordberg, K. ECCV 2018. 26 | 27 | Reference implementation available on the [author github repository][lambda-twist-github]. 28 | 29 | [lambda-twist-github]: https://github.com/midjji/lambdatwist-p3p 30 | 31 | ## Documentation 32 | 33 | To build the documentation with math formatted by katex: 34 | 35 | ```bash 36 | RUSTDOCFLAGS="--html-in-header katex-header.html" cargo doc --no-deps 37 | ``` 38 | 39 | ## License 40 | 41 | This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 42 | If a copy of the MPL was not distributed with this file, 43 | You can obtain one at . 44 | 45 | This rewrite is based on the adaptation of the original code (GPL-3.0) 46 | into [OpenMVG, published under MPL-2.0 with the original author agreement][p3p-openmvg]. 47 | 48 | [p3p-openmvg]: https://github.com/openMVG/openMVG/pull/1500 49 | -------------------------------------------------------------------------------- /tutorial-code/chapter2-first-program/src/main.rs: -------------------------------------------------------------------------------- 1 | use cv::image::{ 2 | image::{self, DynamicImage, Rgba}, 3 | imageproc::drawing, 4 | }; 5 | use rand::Rng; 6 | 7 | fn main() { 8 | // Load the image. 9 | let src_image = image::open("res/0000000000.png").expect("failed to open image file"); 10 | 11 | // Create an RNG so we can draw random points on the image. 12 | let mut rng = rand::thread_rng(); 13 | 14 | // Make a canvas with the `imageproc::drawing` module. 15 | // We use the blend mode so that we can draw with translucency on the image. 16 | // We convert the image to rgba8 during this process. 17 | let mut image_canvas = drawing::Blend(src_image.to_rgba8()); 18 | 19 | // Loop 50 times. 20 | for _ in 0..5000 { 21 | // Generate a random pixel coordinate on the image. 22 | let x = rng.gen_range(0..src_image.width()) as i32; 23 | let y = rng.gen_range(0..src_image.height()) as i32; 24 | // Use the `imageproc::drawing` module to render a cross on the image. 25 | drawing::draw_cross_mut(&mut image_canvas, Rgba([0, 255, 255, 128]), x, y); 26 | } 27 | 28 | // Get the resulting image. 29 | let out_image = DynamicImage::ImageRgba8(image_canvas.0); 30 | 31 | // Save the image to a temporary file. 32 | let image_file_path = tempfile::Builder::new() 33 | .suffix(".png") 34 | .tempfile() 35 | .unwrap() 36 | .into_temp_path(); 37 | out_image.save(&image_file_path).unwrap(); 38 | 39 | // Open the image with the system's default application. 40 | open::that(&image_file_path).unwrap(); 41 | // Some applications may spawn in the background and take a while to begin opening the image, 42 | // and it isn't clear if its possible to always detect whether the child process has been closed. 43 | std::thread::sleep(std::time::Duration::from_secs(5)); 44 | } 45 | -------------------------------------------------------------------------------- /akaze/benches/criterion.rs: -------------------------------------------------------------------------------- 1 | use akaze::Akaze; 2 | use criterion::{criterion_group, criterion_main, Criterion}; 3 | 4 | fn load_image() -> akaze::image::GrayFloatImage { 5 | akaze::image::GrayFloatImage::from_dynamic(&image::open("../res/0000000000.png").unwrap()) 6 | } 7 | 8 | fn extract(c: &mut Criterion) { 9 | let image = load_image(); 10 | let akaze = Akaze::sparse(); 11 | c.bench_function("extract", |b| { 12 | b.iter(|| akaze.extract_from_gray_float_image(&image)) 13 | }); 14 | } 15 | 16 | criterion_group!( 17 | name = akaze; 18 | config = Criterion::default().sample_size(10); 19 | targets = extract 20 | ); 21 | 22 | fn bench_horizontal_filter(c: &mut Criterion) { 23 | let image = load_image(); 24 | let small_kernel = akaze::image::gaussian_kernel(1.0, 7); 25 | c.bench_function("horizontal_filter_small_kernel", |b| { 26 | b.iter(|| akaze::image::horizontal_filter(&image.0, &small_kernel)) 27 | }); 28 | let large_kernel = akaze::image::gaussian_kernel(10.0, 71); 29 | c.bench_function("horizontal_filter_large_kernel", |b| { 30 | b.iter(|| akaze::image::horizontal_filter(&image.0, &large_kernel)) 31 | }); 32 | } 33 | 34 | fn bench_vertical_filter(c: &mut Criterion) { 35 | let image = load_image(); 36 | let small_kernel = akaze::image::gaussian_kernel(1.0, 7); 37 | c.bench_function("vertical_filter_small_kernel", |b| { 38 | b.iter(|| akaze::image::vertical_filter(&image.0, &small_kernel)) 39 | }); 40 | let large_kernel = akaze::image::gaussian_kernel(10.0, 71); 41 | c.bench_function("vertical_filter_large_kernel", |b| { 42 | b.iter(|| akaze::image::vertical_filter(&image.0, &large_kernel)) 43 | }); 44 | } 45 | 46 | criterion_group!( 47 | name = akaze_image; 48 | config = Criterion::default().sample_size(10); 49 | targets = bench_horizontal_filter, bench_vertical_filter 50 | ); 51 | 52 | criterion_main!(akaze, akaze_image); 53 | -------------------------------------------------------------------------------- /tutorial-code/chapter3-akaze-feature-extraction/src/main.rs: -------------------------------------------------------------------------------- 1 | use cv::{ 2 | feature::akaze::{Akaze, KeyPoint}, 3 | image::{ 4 | image::{self, DynamicImage, Rgba}, 5 | imageproc::drawing, 6 | }, 7 | }; 8 | 9 | fn main() { 10 | // Load the image. 11 | let src_image = image::open("res/0000000000.png").expect("failed to open image file"); 12 | 13 | // Create an instance of `Akaze` with the default settings. 14 | let akaze = Akaze::default(); 15 | 16 | // Extract the features from the image using akaze. 17 | let (key_points, _descriptors) = akaze.extract(&src_image); 18 | 19 | // Make a canvas with the `imageproc::drawing` module. 20 | // We use the blend mode so that we can draw with translucency on the image. 21 | // We convert the image to rgba8 during this process. 22 | let mut image_canvas = drawing::Blend(src_image.to_rgba8()); 23 | 24 | // Draw a cross on the image at every keypoint detected. 25 | for KeyPoint { point: (x, y), .. } in key_points { 26 | drawing::draw_cross_mut( 27 | &mut image_canvas, 28 | Rgba([0, 255, 255, 128]), 29 | x as i32, 30 | y as i32, 31 | ); 32 | } 33 | 34 | // Get the resulting image. 35 | let out_image = DynamicImage::ImageRgba8(image_canvas.0); 36 | 37 | // Save the image to a temporary file. 38 | let image_file_path = tempfile::Builder::new() 39 | .suffix(".png") 40 | .tempfile() 41 | .unwrap() 42 | .into_temp_path(); 43 | out_image.save(&image_file_path).unwrap(); 44 | 45 | // Open the image with the system's default application. 46 | open::that(&image_file_path).unwrap(); 47 | // Some applications may spawn in the background and take a while to begin opening the image, 48 | // and it isn't clear if its possible to always detect whether the child process has been closed. 49 | std::thread::sleep(std::time::Duration::from_secs(5)); 50 | } 51 | -------------------------------------------------------------------------------- /akaze/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "akaze" 3 | version = "0.7.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "AKAZE feature extraction algorithm for computer vision" 7 | keywords = ["keypoint", "descriptor", "vision", "sfm", "slam"] 8 | categories = ["computer-vision", "science::robotics"] 9 | repository = "https://github.com/rust-cv/cv" 10 | documentation = "https://docs.rs/akaze/" 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [dependencies] 15 | cv-core = { version = "0.15.0", path = "../cv-core" } 16 | image = { version = "0.25", default-features = false } 17 | log = { version = "0.4.14", default-features = false } 18 | primal = { version = "0.3.0", default-features = false } 19 | derive_more = { version = "0.99.17", default-features = false } 20 | nshare = { version = "0.10", default-features = false, features = [ 21 | "ndarray", 22 | "image", 23 | ] } 24 | ndarray = { version = "0.16", default-features = false } 25 | float-ord = { version = "0.3.2", default-features = false } 26 | space = "0.17.0" 27 | bitarray = "0.9.0" 28 | thiserror = { version = "1.0.40", default-features = false } 29 | wide = "0.7" 30 | serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } 31 | rayon = { version = "1.7.0", optional = true } 32 | 33 | [target.'cfg(target_arch = "wasm32")'.dependencies] 34 | web-sys = { version = "0.3.64", features = ["Window", "Performance"], optional = true } 35 | 36 | [dev-dependencies] 37 | eight-point = { version = "0.8.0", path = "../eight-point" } 38 | cv-pinhole = { version = "0.6.0", path = "../cv-pinhole" } 39 | arrsac = "0.10.0" 40 | rand = "0.8.4" 41 | rand_pcg = "0.3.1" 42 | criterion = "0.3.5" 43 | pretty_env_logger = "0.4.0" 44 | image = "0.25" 45 | bitarray = { version = "0.9.0", features = ["space"] } 46 | imageproc = "0.25.0" 47 | 48 | [[bench]] 49 | name = "criterion" 50 | harness = false 51 | 52 | [features] 53 | serde = ["dep:serde", "bitarray/serde"] 54 | -------------------------------------------------------------------------------- /tutorial/src/chapter1-introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction to Rust-CV 2 | 3 | Rust CV is a project that aims to bring Computer Vision (CV) algorithms to the Rust programming language. To follow this tutorial a basic understanding of Rust, and its ecosystem, is required. Also, computer vision knowledge would help. 4 | 5 | ## About this book 6 | 7 | This tutorial aims to help you understand how to use Rust-CV and what it has to offer. In is current state, this tutorial is rather incomplete and a lot of examples are missing. If you spot an error or wish to add a page feel free to do a PR. We are very open so don't hesitate to contribute. 8 | 9 | The code examples in this book can be found [here](https://github.com/rust-cv/cv/tree/master/tutorial-code/). The source for the book can be found [here](https://github.com/rust-cv/cv/tree/master/tutorial/). 10 | 11 | ## Project structure 12 | 13 | Before using Rust-CV it is important to understand how the ecosystem is set-up and how to use it. 14 | 15 | If you look at the [repository](https://github.com/rust-cv/cv) you can see multiples directories and a `Cargo.toml` file that contains a `[workspace]` section. As the name implies, we are using the workspace feature of Cargo. To put it simply, a cargo workspace allows us to create multiple packages in the same repository. 16 | You can check the [official documentation](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html) to get more details. 17 | 18 | This is what happens here. Rust-CV is a mono-repository project which is made of multiple packages. For instance, we have the `cv-core` directory that contains the very basic things for computer vision. We also have the `imgshow` directory which allow us to show images. There is many more but we won't go deeper for now. The `cv` crate needs to be explained though. The `cv` crate is rather empty (code wise) and just reexports most of the other package so by just depending on it we have most of what the project has to offer. 19 | 20 | There are three things to remember here : 21 | * [Rust-CV repository](https://github.com/rust-cv/cv) is a mono repository that is split-up into multiple packages. 22 | * The way the project is structured allows you to use tiny crates so you don't have to pay the price for all the CV code if you just use a subpart. 23 | * The project defines a crate named `cv` that depends on many others just to re-export them. This is useful to get started faster, as by just pulling this crate you have already many computer vision algorithms and data structures ready to use. 24 | -------------------------------------------------------------------------------- /akaze/src/contrast_factor.rs: -------------------------------------------------------------------------------- 1 | use crate::image::{gaussian_blur, GrayFloatImage}; 2 | use log::*; 3 | 4 | /// This function computes a good empirical value for the k contrast factor 5 | /// given an input image, the percentile (0-1), the gradient scale and the 6 | /// number of bins in the histogram. 7 | /// 8 | /// # Arguments 9 | /// * `image` Input imagm 10 | /// * `percentile` - Percentile of the image gradient histogram (0-1) 11 | /// * `gradient_histogram_scale` - Scale for computing the image gradient histogram 12 | /// * `nbins` - Number of histogram bins 13 | /// # Return value 14 | /// k contrast factor 15 | #[allow(non_snake_case)] 16 | pub fn compute_contrast_factor( 17 | image: &GrayFloatImage, 18 | percentile: f64, 19 | gradient_histogram_scale: f64, 20 | num_bins: usize, 21 | ) -> f64 { 22 | let mut num_points: f64 = 0.0; 23 | let mut histogram = vec![0; num_bins]; 24 | let gaussian = gaussian_blur(image, gradient_histogram_scale as f32); 25 | let Lx = crate::derivatives::simple_scharr_horizontal(&gaussian); 26 | let Ly = crate::derivatives::simple_scharr_vertical(&gaussian); 27 | let hmax = (1..gaussian.height() - 1) 28 | .flat_map(|y| (1..gaussian.width() - 1).map(move |x| (x, y))) 29 | .map(|(x, y)| Lx.get(x, y).powi(2) as f64 + Ly.get(x, y).powi(2) as f64) 30 | .map(float_ord::FloatOrd) 31 | .max() 32 | .unwrap() 33 | .0 34 | .sqrt(); 35 | for y in 1..(gaussian.height() - 1) { 36 | for x in 1..(gaussian.width() - 1) { 37 | let modg = (Lx.get(x, y).powi(2) as f64 + Ly.get(x, y).powi(2) as f64).sqrt(); 38 | if modg != 0.0 { 39 | let mut bin_number = f64::floor((num_bins as f64) * (modg / hmax)) as usize; 40 | if bin_number == num_bins { 41 | bin_number -= 1; 42 | } 43 | histogram[bin_number] += 1; 44 | num_points += 1f64; 45 | } 46 | } 47 | } 48 | let threshold: usize = (num_points * percentile) as usize; 49 | let mut k: usize = 0; 50 | let mut num_elements: usize = 0; 51 | while num_elements < threshold && k < num_bins { 52 | num_elements += histogram[k]; 53 | k += 1; 54 | } 55 | debug!( 56 | "hmax: {}, threshold: {}, num_elements: {}", 57 | hmax, threshold, num_elements 58 | ); 59 | if num_elements >= threshold { 60 | hmax * (k as f64) / (num_bins as f64) 61 | } else { 62 | 0.03 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /cv-sfm/src/bicubic.rs: -------------------------------------------------------------------------------- 1 | //! Note: This code came directly from the imageproc code base because it was private code. 2 | //! I could not find an alternative implementation for the image crate. This will likely 3 | //! go away once we find a library that includes bicubic sampling, or if the image 4 | //! crate itself includes it. 5 | 6 | use conv::ValueInto; 7 | use image::{GenericImageView, Pixel}; 8 | use imageproc::definitions::{Clamp, Image}; 9 | 10 | fn blend_cubic

(px0: &P, px1: &P, px2: &P, px3: &P, x: f32) -> P 11 | where 12 | P: Pixel, 13 | P::Subpixel: ValueInto + Clamp, 14 | { 15 | let mut outp = *px0; 16 | 17 | for i in 0..(P::CHANNEL_COUNT as usize) { 18 | let p0 = px0.channels()[i].value_into().unwrap(); 19 | let p1 = px1.channels()[i].value_into().unwrap(); 20 | let p2 = px2.channels()[i].value_into().unwrap(); 21 | let p3 = px3.channels()[i].value_into().unwrap(); 22 | #[rustfmt::skip] 23 | let pval = p1 + 0.5 * x * (p2 - p0 + x * (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3 + x * (3.0 * (p1 - p2) + p3 - p0))); 24 | outp.channels_mut()[i] =

::Subpixel::clamp(pval); 25 | } 26 | 27 | outp 28 | } 29 | 30 | pub fn interpolate_bicubic

(image: &Image

, x: f32, y: f32, default: P) -> P 31 | where 32 | P: Pixel + 'static, 33 |

::Subpixel: ValueInto + Clamp, 34 | { 35 | let left = x.floor() - 1f32; 36 | let right = left + 4f32; 37 | let top = y.floor() - 1f32; 38 | let bottom = top + 4f32; 39 | 40 | let x_weight = x - (left + 1f32); 41 | let y_weight = y - (top + 1f32); 42 | 43 | let mut col: [P; 4] = [default, default, default, default]; 44 | 45 | let (width, height) = image.dimensions(); 46 | if left < 0f32 || right >= width as f32 || top < 0f32 || bottom >= height as f32 { 47 | default 48 | } else { 49 | for row in top as u32..bottom as u32 { 50 | let (p0, p1, p2, p3): (P, P, P, P) = unsafe { 51 | ( 52 | image.unsafe_get_pixel(left as u32, row), 53 | image.unsafe_get_pixel(left as u32 + 1, row), 54 | image.unsafe_get_pixel(left as u32 + 2, row), 55 | image.unsafe_get_pixel(left as u32 + 3, row), 56 | ) 57 | }; 58 | 59 | let c = blend_cubic(&p0, &p1, &p2, &p3, x_weight); 60 | col[row as usize - top as usize] = c; 61 | } 62 | 63 | blend_cubic(&col[0], &col[1], &col[2], &col[3], y_weight) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /cv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cv" 3 | version = "0.6.0" 4 | authors = ["Geordon Worley "] 5 | edition = "2021" 6 | description = "Batteries-included pure-Rust computer vision crate" 7 | keywords = ["computer", "vision", "photogrammetry", "camera"] 8 | categories = ["computer-vision"] 9 | repository = "https://github.com/rust-cv/cv" 10 | documentation = "https://docs.rs/cv/" 11 | license = "MIT" 12 | readme = "README.md" 13 | 14 | [features] 15 | default = [ 16 | "alloc", 17 | "cv-pinhole", 18 | "cv-geom", 19 | #"cv-sfm", 20 | "eight-point", 21 | "nister-stewenius", 22 | "lambda-twist", 23 | "akaze", 24 | "space", 25 | "hnsw", 26 | "levenberg-marquardt", 27 | "arrsac", 28 | "bitarray", 29 | "image", 30 | "imageproc", 31 | "jpeg", 32 | "png", 33 | # eye is not added by default because it requires libclang-dev to be installed to build. 34 | # If we can figure out how to get it to build without system dependencies, it can be made default. 35 | # "eye", 36 | "ndarray-vision", 37 | ] 38 | alloc = ["cv-pinhole/alloc", "space/alloc"] 39 | serde-serialize = [ 40 | "cv-core/serde-serialize", 41 | "cv-pinhole/serde-serialize", 42 | "cv-sfm/serde-serialize", 43 | ] 44 | jpeg = ["image", "image/jpeg"] 45 | png = ["image", "image/png"] 46 | 47 | [dependencies] 48 | cv-core = { version = "0.15.0", path = "../cv-core" } 49 | cv-pinhole = { optional = true, version = "0.6.0", path = "../cv-pinhole" } 50 | cv-geom = { optional = true, version = "0.7.0", path = "../cv-geom" } 51 | cv-sfm = { optional = true, version = "0.1.0", path = "../cv-sfm" } 52 | eight-point = { optional = true, version = "0.8.0", path = "../eight-point" } 53 | nister-stewenius = { optional = true, version = "0.1.0", path = "../nister-stewenius" } 54 | lambda-twist = { optional = true, version = "0.7.0", path = "../lambda-twist" } 55 | akaze = { optional = true, version = "0.7.0", path = "../akaze" } 56 | space = { version = "0.17.0", optional = true } 57 | hnsw = { version = "0.11.0", optional = true } 58 | hgg = { version = "0.4.1", optional = true } 59 | levenberg-marquardt = { version = "0.12.0", optional = true } 60 | arrsac = { version = "0.10.0", optional = true } 61 | bitarray = { version = "0.9.0", features = ["space"], optional = true } 62 | image = { version = "0.25", default-features = false, optional = true } 63 | imageproc = { version = "0.25", default-features = false, optional = true } 64 | eye = { version = "0.4.1", optional = true } 65 | ndarray-vision = { version = "0.3.0", default-features = false, optional = true } 66 | 67 | [package.metadata.docs.rs] 68 | all-features = true 69 | -------------------------------------------------------------------------------- /eight-point/tests/random.rs: -------------------------------------------------------------------------------- 1 | use cv_core::{ 2 | nalgebra::{IsometryMatrix3, Point3, Rotation3, UnitVector3, Vector3}, 3 | sample_consensus::Model, 4 | CameraPoint, CameraToCamera, FeatureMatch, Pose, Projective, 5 | }; 6 | 7 | const SAMPLE_POINTS: usize = 16; 8 | const RESIDUAL_THRESHOLD: f64 = 1e-4; 9 | 10 | const ROT_MAGNITUDE: f64 = 0.2; 11 | const POINT_BOX_SIZE: f64 = 2.0; 12 | const POINT_DISTANCE: f64 = 3.0; 13 | 14 | #[test] 15 | fn randomized() { 16 | let successes = (0..1000).filter(|_| run_round()).count(); 17 | eprintln!("successes: {}", successes); 18 | assert!(successes > 950); 19 | } 20 | 21 | fn run_round() -> bool { 22 | let mut success = true; 23 | let (_, aps, bps) = some_test_data(); 24 | let matches = aps.iter().zip(&bps).map(|(&a, &b)| FeatureMatch(a, b)); 25 | let eight_point = eight_point::EightPoint::new(); 26 | let essential = eight_point 27 | .from_matches(matches.clone()) 28 | .expect("didn't get any essential matrix"); 29 | for m in matches.clone() { 30 | if essential.residual(&m).abs() > RESIDUAL_THRESHOLD { 31 | success = false; 32 | eprintln!("failed residual check: {}", essential.residual(&m).abs()); 33 | } 34 | } 35 | success 36 | } 37 | 38 | /// Gets a random relative pose, input points A, input points B, and A point depths. 39 | fn some_test_data() -> ( 40 | CameraToCamera, 41 | [UnitVector3; SAMPLE_POINTS], 42 | [UnitVector3; SAMPLE_POINTS], 43 | ) { 44 | // The relative pose orientation is fixed and translation is random. 45 | let relative_pose = CameraToCamera(IsometryMatrix3::from_parts( 46 | Vector3::new_random().into(), 47 | Rotation3::new(Vector3::new_random() * std::f64::consts::PI * 2.0 * ROT_MAGNITUDE), 48 | )); 49 | 50 | // Generate A's camera points. 51 | let cams_a = (0..SAMPLE_POINTS) 52 | .map(|_| { 53 | let mut a = Point3::from(Vector3::new_random() * POINT_BOX_SIZE); 54 | a.x -= 0.5 * POINT_BOX_SIZE; 55 | a.y -= 0.5 * POINT_BOX_SIZE; 56 | a.z += POINT_DISTANCE; 57 | CameraPoint::from_point(a) 58 | }) 59 | .collect::>() 60 | .into_iter(); 61 | 62 | // Generate B's camera points. 63 | let cams_b = cams_a.clone().map(|a| relative_pose.transform(a)); 64 | 65 | let mut kps_a = [UnitVector3::new_normalize(Vector3::z()); SAMPLE_POINTS]; 66 | for (keypoint, camera) in kps_a.iter_mut().zip(cams_a) { 67 | *keypoint = camera.bearing(); 68 | } 69 | let mut kps_b = [UnitVector3::new_normalize(Vector3::z()); SAMPLE_POINTS]; 70 | for (keypoint, camera) in kps_b.iter_mut().zip(cams_b.clone()) { 71 | *keypoint = camera.bearing(); 72 | } 73 | 74 | (relative_pose, kps_a, kps_b) 75 | } 76 | -------------------------------------------------------------------------------- /eight-point/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | 3 | use arrayvec::ArrayVec; 4 | use cv_core::{ 5 | nalgebra::{self, Matrix3, OMatrix, OVector, U8, U9}, 6 | sample_consensus::Estimator, 7 | CameraToCamera, FeatureMatch, 8 | }; 9 | use cv_pinhole::EssentialMatrix; 10 | 11 | fn encode_epipolar_equation(matches: impl Iterator) -> OMatrix { 12 | let mut out: OMatrix = nalgebra::zero(); 13 | for (i, FeatureMatch(a, b)) in (0..8).zip(matches) { 14 | let mut row = OVector::::zeros(); 15 | let ap = a.into_inner() / a.z; 16 | let bp = b.into_inner() / a.z; 17 | for j in 0..3 { 18 | let v = ap[j] * bp; 19 | row.fixed_rows_mut::<3>(3 * j).copy_from(&v); 20 | } 21 | out.row_mut(i).copy_from(&row.transpose()); 22 | } 23 | out 24 | } 25 | 26 | /// Performs the 27 | /// [eight-point algorithm](https://en.wikipedia.org/wiki/Eight-point_algorithm) 28 | /// by Richard Hartley and Andrew Zisserman. 29 | /// 30 | /// To recondition the matrix produced by estimation, see 31 | /// [`cv_pinhole::EssentialMatrix::recondition`]. 32 | #[derive(Copy, Clone, Debug)] 33 | pub struct EightPoint { 34 | pub epsilon: f64, 35 | pub iterations: usize, 36 | } 37 | 38 | impl EightPoint { 39 | pub fn new() -> Self { 40 | Default::default() 41 | } 42 | 43 | pub fn from_matches(&self, data: I) -> Option 44 | where 45 | I: Iterator + Clone, 46 | { 47 | let epipolar_constraint = encode_epipolar_equation(data); 48 | let eet = epipolar_constraint.transpose() * epipolar_constraint; 49 | let eigens = eet.try_symmetric_eigen(self.epsilon, self.iterations)?; 50 | let eigenvector = eigens 51 | .eigenvalues 52 | .iter() 53 | .enumerate() 54 | .min_by_key(|&(_, &n)| float_ord::FloatOrd(n)) 55 | .map(|(ix, _)| eigens.eigenvectors.column(ix).into_owned())?; 56 | let mat = Matrix3::from_iterator(eigenvector.iter().copied()); 57 | Some(EssentialMatrix(mat)) 58 | } 59 | } 60 | 61 | impl Default for EightPoint { 62 | fn default() -> Self { 63 | Self { 64 | epsilon: 1e-12, 65 | iterations: 1000, 66 | } 67 | } 68 | } 69 | 70 | impl Estimator for EightPoint { 71 | type Model = CameraToCamera; 72 | type ModelIter = ArrayVec; 73 | const MIN_SAMPLES: usize = 8; 74 | 75 | fn estimate(&self, data: I) -> Self::ModelIter 76 | where 77 | I: Iterator + Clone, 78 | { 79 | self.from_matches(data) 80 | .and_then(|essential| essential.possible_unscaled_poses(self.epsilon, self.iterations)) 81 | .map(Into::into) 82 | .unwrap_or_default() 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /akaze/src/derivatives.rs: -------------------------------------------------------------------------------- 1 | use crate::image::{separable_filter, GrayFloatImage}; 2 | 3 | pub fn simple_scharr_horizontal(image: &GrayFloatImage) -> GrayFloatImage { 4 | // similar to cv::Scharr with xorder=1, yorder=0, scale=1, delta=0 5 | GrayFloatImage(separable_filter(&image.0, &[-1., 0., 1.], &[3., 10., 3.])) 6 | } 7 | 8 | pub fn simple_scharr_vertical(image: &GrayFloatImage) -> GrayFloatImage { 9 | // similar to cv::Scharr with xorder=0, yorder=1, scale=1, delta=0 10 | GrayFloatImage(separable_filter(&image.0, &[3., 10., 3.], &[-1., 0., 1.])) 11 | } 12 | 13 | /// Compute the Scharr derivative horizontally 14 | /// 15 | /// The implementation of this function is using a separable kernel, for speed. 16 | /// 17 | /// # Arguments 18 | /// * `image` - the input image. 19 | /// * `sigma_size` - the scale of the derivative. 20 | /// 21 | /// # Return value 22 | /// Output image derivative (an image.) 23 | pub fn scharr_horizontal(image: &GrayFloatImage, sigma_size: u32) -> GrayFloatImage { 24 | if sigma_size == 1 { 25 | return simple_scharr_horizontal(image); 26 | } 27 | let main_kernel = computer_scharr_kernel(sigma_size, FilterOrder::Main); 28 | let off_kernel = computer_scharr_kernel(sigma_size, FilterOrder::Off); 29 | GrayFloatImage(separable_filter(&image.0, &main_kernel, &off_kernel)) 30 | } 31 | 32 | /// Compute the Scharr derivative vertically 33 | /// 34 | /// The implementation of this function is using a separable kernel, for speed. 35 | /// 36 | /// # Arguments 37 | /// * `image` - the input image. 38 | /// * `sigma_size` - the scale of the derivative. 39 | /// 40 | /// # Return value 41 | /// Output image derivative (an image.) 42 | pub fn scharr_vertical(image: &GrayFloatImage, sigma_size: u32) -> GrayFloatImage { 43 | if sigma_size == 1 { 44 | return simple_scharr_vertical(image); 45 | } 46 | let main_kernel = computer_scharr_kernel(sigma_size, FilterOrder::Main); 47 | let off_kernel = computer_scharr_kernel(sigma_size, FilterOrder::Off); 48 | GrayFloatImage(separable_filter(&image.0, &off_kernel, &main_kernel)) 49 | } 50 | 51 | #[derive(Copy, Clone, Debug, PartialEq)] 52 | enum FilterOrder { 53 | Main, 54 | Off, 55 | } 56 | 57 | fn computer_scharr_kernel(sigma_size: u32, order: FilterOrder) -> Vec { 58 | // Difference between middle and sides of main axis filter. 59 | let w = 10.0 / 3.0; 60 | // Side intensity of filter. 61 | let norm = (1.0 / (2.0 * f64::from(sigma_size) * (w + 2.0))) as f32; 62 | // Middle intensity of filter. 63 | let middle = norm * w as f32; 64 | // Size of kernel 65 | let ksize = (3 + 2 * (sigma_size - 1)) as usize; 66 | let mut kernel = vec![0.0; ksize]; 67 | match order { 68 | FilterOrder::Main => { 69 | kernel[0] = -1.0; 70 | kernel[ksize - 1] = 1.0; 71 | } 72 | FilterOrder::Off => { 73 | kernel[0] = norm; 74 | kernel[ksize / 2] = middle; 75 | kernel[ksize - 1] = norm; 76 | } 77 | }; 78 | kernel 79 | } 80 | -------------------------------------------------------------------------------- /cv-core/src/triangulation.rs: -------------------------------------------------------------------------------- 1 | use crate::{CameraPoint, CameraToCamera, Pose, Projective, WorldPoint, WorldToCamera}; 2 | use nalgebra::UnitVector3; 3 | 4 | /// This trait is for algorithms which allow you to triangulate a point from two or more observances. 5 | /// Each observance is a [`WorldToCamera`] and a bearing. 6 | /// 7 | /// Returned points will always be checked successfully for chirality. 8 | pub trait TriangulatorObservations { 9 | /// This function takes a series of [`WorldToCamera`] and bearings that (supposedly) correspond to the same 3d point. 10 | /// It returns the triangulated [`WorldPoint`] if successful. 11 | fn triangulate_observations( 12 | &self, 13 | pairs: impl Iterator)> + Clone, 14 | ) -> Option; 15 | 16 | /// This function takes one bearing (`center_bearing`) coming from the camera whos reference frame we will 17 | /// triangulate the [`CameraPoint`] in and an iterator over a series of observations 18 | /// from other cameras, along with the transformation from the original camera to the observation's camera. 19 | /// It returns the triangulated [`CameraPoint`] if successful. 20 | #[inline(always)] 21 | fn triangulate_observations_to_camera( 22 | &self, 23 | center_bearing: UnitVector3, 24 | pairs: impl Iterator)> + Clone, 25 | ) -> Option { 26 | use core::iter::once; 27 | 28 | // We use the first camera as the "world", thus it is the identity (optical center at origin). 29 | // The each subsequent pose maps the first camera (the world) to the second camera (the camera). 30 | // This is how we convert the `CameraToCamera` into a `WorldToCamera`. 31 | self.triangulate_observations( 32 | once((WorldToCamera::identity(), center_bearing)) 33 | .chain(pairs.map(|(pose, bearing)| (WorldToCamera(pose.0), bearing))), 34 | ) 35 | .map(|p| CameraPoint::from_homogeneous(p.0)) 36 | } 37 | } 38 | 39 | /// This trait allows you to take one relative pose from camera `A` to camera `B` and two bearings `a` and `b` from 40 | /// their respective cameras to triangulate a point from the perspective of camera `A`. 41 | /// 42 | /// Returned points will always be checked successfully for chirality. 43 | pub trait TriangulatorRelative { 44 | fn triangulate_relative( 45 | &self, 46 | relative_pose: CameraToCamera, 47 | a: UnitVector3, 48 | b: UnitVector3, 49 | ) -> Option; 50 | } 51 | 52 | impl TriangulatorRelative for T 53 | where 54 | T: TriangulatorObservations, 55 | { 56 | #[inline(always)] 57 | fn triangulate_relative( 58 | &self, 59 | pose: CameraToCamera, 60 | a: UnitVector3, 61 | b: UnitVector3, 62 | ) -> Option { 63 | use core::iter::once; 64 | 65 | self.triangulate_observations_to_camera(a, once((pose, b))) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /res/calib_cam_to_cam.txt: -------------------------------------------------------------------------------- 1 | calib_time: 09-Jan-2012 13:57:47 2 | corner_dist: 9.950000e-02 3 | S_00: 1.392000e+03 5.120000e+02 4 | K_00: 9.842439e+02 0.000000e+00 6.900000e+02 0.000000e+00 9.808141e+02 2.331966e+02 0.000000e+00 0.000000e+00 1.000000e+00 5 | D_00: -3.728755e-01 2.037299e-01 2.219027e-03 1.383707e-03 -7.233722e-02 6 | R_00: 1.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 0.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 7 | T_00: 2.573699e-16 -1.059758e-16 1.614870e-16 8 | S_rect_00: 1.242000e+03 3.750000e+02 9 | R_rect_00: 9.999239e-01 9.837760e-03 -7.445048e-03 -9.869795e-03 9.999421e-01 -4.278459e-03 7.402527e-03 4.351614e-03 9.999631e-01 10 | P_rect_00: 7.215377e+02 0.000000e+00 6.095593e+02 0.000000e+00 0.000000e+00 7.215377e+02 1.728540e+02 0.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 0.000000e+00 11 | S_01: 1.392000e+03 5.120000e+02 12 | K_01: 9.895267e+02 0.000000e+00 7.020000e+02 0.000000e+00 9.878386e+02 2.455590e+02 0.000000e+00 0.000000e+00 1.000000e+00 13 | D_01: -3.644661e-01 1.790019e-01 1.148107e-03 -6.298563e-04 -5.314062e-02 14 | R_01: 9.993513e-01 1.860866e-02 -3.083487e-02 -1.887662e-02 9.997863e-01 -8.421873e-03 3.067156e-02 8.998467e-03 9.994890e-01 15 | T_01: -5.370000e-01 4.822061e-03 -1.252488e-02 16 | S_rect_01: 1.242000e+03 3.750000e+02 17 | R_rect_01: 9.996878e-01 -8.976826e-03 2.331651e-02 8.876121e-03 9.999508e-01 4.418952e-03 -2.335503e-02 -4.210612e-03 9.997184e-01 18 | P_rect_01: 7.215377e+02 0.000000e+00 6.095593e+02 -3.875744e+02 0.000000e+00 7.215377e+02 1.728540e+02 0.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 0.000000e+00 19 | S_02: 1.392000e+03 5.120000e+02 20 | K_02: 9.597910e+02 0.000000e+00 6.960217e+02 0.000000e+00 9.569251e+02 2.241806e+02 0.000000e+00 0.000000e+00 1.000000e+00 21 | D_02: -3.691481e-01 1.968681e-01 1.353473e-03 5.677587e-04 -6.770705e-02 22 | R_02: 9.999758e-01 -5.267463e-03 -4.552439e-03 5.251945e-03 9.999804e-01 -3.413835e-03 4.570332e-03 3.389843e-03 9.999838e-01 23 | T_02: 5.956621e-02 2.900141e-04 2.577209e-03 24 | S_rect_02: 1.242000e+03 3.750000e+02 25 | R_rect_02: 9.998817e-01 1.511453e-02 -2.841595e-03 -1.511724e-02 9.998853e-01 -9.338510e-04 2.827154e-03 9.766976e-04 9.999955e-01 26 | P_rect_02: 7.215377e+02 0.000000e+00 6.095593e+02 4.485728e+01 0.000000e+00 7.215377e+02 1.728540e+02 2.163791e-01 0.000000e+00 0.000000e+00 1.000000e+00 2.745884e-03 27 | S_03: 1.392000e+03 5.120000e+02 28 | K_03: 9.037596e+02 0.000000e+00 6.957519e+02 0.000000e+00 9.019653e+02 2.242509e+02 0.000000e+00 0.000000e+00 1.000000e+00 29 | D_03: -3.639558e-01 1.788651e-01 6.029694e-04 -3.922424e-04 -5.382460e-02 30 | R_03: 9.995599e-01 1.699522e-02 -2.431313e-02 -1.704422e-02 9.998531e-01 -1.809756e-03 2.427880e-02 2.223358e-03 9.997028e-01 31 | T_03: -4.731050e-01 5.551470e-03 -5.250882e-03 32 | S_rect_03: 1.242000e+03 3.750000e+02 33 | R_rect_03: 9.998321e-01 -7.193136e-03 1.685599e-02 7.232804e-03 9.999712e-01 -2.293585e-03 -1.683901e-02 2.415116e-03 9.998553e-01 34 | P_rect_03: 7.215377e+02 0.000000e+00 6.095593e+02 -3.395242e+02 0.000000e+00 7.215377e+02 1.728540e+02 2.199936e+00 0.000000e+00 0.000000e+00 1.000000e+00 2.729905e-03 35 | -------------------------------------------------------------------------------- /akaze/src/nonlinear_diffusion.rs: -------------------------------------------------------------------------------- 1 | use crate::{EvolutionStep, GrayFloatImage}; 2 | use ndarray::{azip, s, Array2}; 3 | 4 | /// This function performs a scalar non-linear diffusion step. 5 | /// 6 | /// # Arguments 7 | /// * `Ld` - Output image in the evolution 8 | /// * `c` - Conductivity image. The function c is a scalar value that depends on the gradient norm 9 | /// * `Lstep` - Previous image in the evolution 10 | /// * `step_size` - The step size in time units 11 | /// Forward Euler Scheme 3x3 stencil 12 | /// dL_by_ds = d(c dL_by_dx)_by_dx + d(c dL_by_dy)_by_dy 13 | #[allow(non_snake_case)] 14 | pub fn calculate_step(evolution_step: &mut EvolutionStep, step_size: f32) { 15 | // Get the ndarray types. 16 | let mut input = evolution_step.Lt.mut_array2(); 17 | let conductivities = evolution_step.Lflow.ref_array2(); 18 | let dim = input.dim(); 19 | // Horizontal flow. 20 | let mut horizontal_flow = Array2::::zeros((dim.0, dim.1 - 1)); 21 | azip!(( 22 | flow in &mut horizontal_flow, 23 | &a in input.slice(s![.., ..-1]), 24 | &b in input.slice(s![.., 1..]), 25 | &ca in conductivities.slice(s![.., ..-1]), 26 | &cb in conductivities.slice(s![.., 1..]), 27 | ) { 28 | *flow = 0.5 * step_size * (ca + cb) * (b - a); 29 | }); 30 | // Vertical flow. 31 | let mut vertical_flow = Array2::::zeros((dim.0 - 1, dim.1)); 32 | azip!(( 33 | flow in &mut vertical_flow, 34 | &a in input.slice(s![..-1, ..]), 35 | &b in input.slice(s![1.., ..]), 36 | &ca in conductivities.slice(s![..-1, ..]), 37 | &cb in conductivities.slice(s![1.., ..]), 38 | ) { 39 | *flow = 0.5 * step_size * (ca + cb) * (b - a); 40 | }); 41 | 42 | // Left 43 | input 44 | .slice_mut(s![.., ..-1]) 45 | .zip_mut_with(&horizontal_flow, |acc, &i| *acc += i); 46 | // Right 47 | input 48 | .slice_mut(s![.., 1..]) 49 | .zip_mut_with(&horizontal_flow, |acc, &i| *acc -= i); 50 | // Up 51 | input 52 | .slice_mut(s![..-1, ..]) 53 | .zip_mut_with(&vertical_flow, |acc, &i| *acc += i); 54 | // Down 55 | input 56 | .slice_mut(s![1.., ..]) 57 | .zip_mut_with(&vertical_flow, |acc, &i| *acc -= i); 58 | } 59 | 60 | /// This function computes the Perona and Malik conductivity coefficient g2 61 | /// g2 = 1 / (1 + dL^2 / k^2) 62 | /// 63 | /// # Arguments 64 | /// * `Lx` - First order image derivative in X-direction (horizontal) 65 | /// * `Ly` - First order image derivative in Y-direction (vertical) 66 | /// * `k` - Contrast factor parameter 67 | /// # Return value 68 | /// Output image 69 | #[allow(non_snake_case)] 70 | pub fn pm_g2(Lx: &GrayFloatImage, Ly: &GrayFloatImage, k: f64) -> GrayFloatImage { 71 | assert!(Lx.width() == Ly.width()); 72 | assert!(Lx.height() == Ly.height()); 73 | let inverse_k = (1.0f64 / (k * k)) as f32; 74 | let mut conductivities = Lx.zero_array(); 75 | azip!(( 76 | c in &mut conductivities, 77 | &x in Lx.ref_array2(), 78 | &y in Ly.ref_array2(), 79 | ) { 80 | *c = 1.0 / (1.0 + inverse_k * (x * x + y * y)); 81 | }); 82 | GrayFloatImage::from_array2(conductivities) 83 | } 84 | -------------------------------------------------------------------------------- /akaze/tests/estimate_pose.rs: -------------------------------------------------------------------------------- 1 | use akaze::Akaze; 2 | use arrsac::Arrsac; 3 | use bitarray::{BitArray, Hamming}; 4 | use cv_core::{ 5 | nalgebra::{Point2, Vector2}, 6 | sample_consensus::Consensus, 7 | CameraModel, FeatureMatch, 8 | }; 9 | use log::*; 10 | use rand::SeedableRng; 11 | use rand_pcg::Pcg64; 12 | use space::Knn; 13 | use std::path::Path; 14 | 15 | const LOWES_RATIO: f32 = 0.5; 16 | 17 | type Descriptor = BitArray<64>; 18 | type Match = FeatureMatch; 19 | 20 | fn image_to_kps(path: impl AsRef) -> (Vec, Vec) { 21 | Akaze::sparse().extract_path(path).unwrap() 22 | } 23 | 24 | #[test] 25 | fn estimate_pose() { 26 | pretty_env_logger::init_timed(); 27 | // Intrinsics retrieved from calib_cam_to_cam.txt K_00. 28 | let intrinsics = cv_pinhole::CameraIntrinsics { 29 | focals: Vector2::new(9.842_439e2, 9.808_141e2), 30 | principal_point: Point2::new(6.9e2, 2.331_966e2), 31 | skew: 0.0, 32 | }; 33 | 34 | // Extract features with AKAZE. 35 | info!("Extracting features"); 36 | let (kps1, ds1) = image_to_kps("../res/0000000000.png"); 37 | let (kps2, ds2) = image_to_kps("../res/0000000014.png"); 38 | 39 | // This ensures the underlying algorithm does not change 40 | // by making sure that we get the exact expected number of features. 41 | assert_eq!(ds1.len(), 399); 42 | assert_eq!(ds2.len(), 343); 43 | 44 | // Perform matching. 45 | info!( 46 | "Running matching on {} and {} descriptors", 47 | ds1.len(), 48 | ds2.len() 49 | ); 50 | let matches: Vec = match_descriptors(&ds1, &ds2) 51 | .into_iter() 52 | .map(|(ix1, ix2)| { 53 | let a = intrinsics.calibrate(kps1[ix1]); 54 | let b = intrinsics.calibrate(kps2[ix2]); 55 | FeatureMatch(a, b) 56 | }) 57 | .collect(); 58 | info!("Finished matching with {} matches", matches.len()); 59 | assert_eq!(matches.len(), 11); 60 | 61 | // Run ARRSAC with the eight-point algorithm. 62 | info!("Running ARRSAC"); 63 | let mut arrsac = Arrsac::new(0.1, Pcg64::from_seed([1; 32])); 64 | let eight_point = eight_point::EightPoint::new(); 65 | let (_, inliers) = arrsac 66 | .model_inliers(&eight_point, matches.iter().copied()) 67 | .expect("failed to estimate model"); 68 | info!("inliers: {}", inliers.len()); 69 | info!( 70 | "inlier ratio: {}", 71 | inliers.len() as f32 / matches.len() as f32 72 | ); 73 | 74 | // Ensures the underlying algorithms don't change at all. 75 | assert_eq!(inliers.len(), 11); 76 | } 77 | 78 | fn match_descriptors(ds1: &[Descriptor], ds2: &[Descriptor]) -> Vec<(usize, usize)> { 79 | let two_neighbors = ds1 80 | .iter() 81 | .map(|d1| { 82 | let neighbors = space::LinearKnn { 83 | metric: Hamming, 84 | iter: ds2.iter(), 85 | } 86 | .knn(d1, 2); 87 | assert_eq!(neighbors.len(), 2, "there should be at least two matches"); 88 | neighbors 89 | }) 90 | .enumerate(); 91 | let satisfies_lowes_ratio = two_neighbors.filter(|(_, neighbors)| { 92 | (neighbors[0].distance as f32) < neighbors[1].distance as f32 * LOWES_RATIO 93 | }); 94 | satisfies_lowes_ratio 95 | .map(|(ix1, neighbors)| (ix1, neighbors[0].index)) 96 | .collect() 97 | } 98 | -------------------------------------------------------------------------------- /cv-core/notes/derivation_of_triangulate_bearing_reproject.md: -------------------------------------------------------------------------------- 1 | This document describes the proof describing the derivation of `cv_core::geom::reproject_along_translation`. 2 | 3 | This function finds a translation from a 3d point that minimizes the reprojection error. 4 | 5 | Here is a key and a diagram: 6 | 7 | - `t` the translation vector 8 | - `b` the `from` point 9 | - `a` the `to` epipolar point 10 | - `O` the optical center 11 | - `@` the virtual image plane 12 | 13 | ``` 14 | t<---b 15 | / 16 | / 17 | @@@a@@@/@@@@@ 18 | | / 19 | | / 20 | |/ 21 | O 22 | ``` 23 | 24 | Note that the direction of vector `t` is not relevant. The translation will be scaled to reduce the reprojection error, even if that means it reverses direction. 25 | 26 | The way this is solved is by starting with the definition of the reprojection error in this case, and to get to that we must define the reprojection. 27 | 28 | First, let `s` scale the translation. 29 | 30 | The reprojection `p` can be defined as follows: 31 | 32 | ``` 33 | c = b + s * t 34 | p = c.xy / c.z 35 | ``` 36 | 37 | This is because `c` is the 3d point after the translation and we can reproject it back to the virtual image plane by dividing the vector by its `z` component to set `z` to `1.0`. Since the `z` component can be discarded to retrieve the normalized image coordinate once it is on the virtual image plane, we don't take this intermediate step and simply divide the `x` and `y` components by `z` to achieve the same result. 38 | 39 | Next, we must compute the `x` and `y` residual. Let `r` be the residual error vector for `x` and `y`. 40 | 41 | ``` 42 | r = a - p 43 | ``` 44 | 45 | This is simply defined as the distance from `p` to `a`. We want to minimize the norm of this vector, and ideally set it to zero. 46 | 47 | ``` 48 | r = <0, 0> 49 | ``` 50 | 51 | If we assume that the residual error can be reduced to zero (which may not be possible), then we can solve for `s` where `r = <0, 0>`. In the following we will derive `s` and add all priors: 52 | 53 | ``` 54 | c = b + s * t 55 | p = c.xy / c.z 56 | r = a - p 57 | r = <0, 0> 58 | 59 | a - p = <0, 0> 60 | = <0, 0> 61 | 62 | a.x - p.x = 0 63 | a.y - p.y = 0 64 | 65 | p.x = (b.x + s * t.x) / (b.z + s * t.z) 66 | p.y = (b.y + s * t.y) / (b.z + s * t.z) 67 | 68 | 0 = a.x - (b.x + s * t.x) / (b.z + s * t.z) 69 | 0 = a.y - (b.y + s * t.y) / (b.z + s * t.z) 70 | 71 | a.x = (b.x + s * t.x) / (b.z + s * t.z) 72 | a.y = (b.y + s * t.y) / (b.z + s * t.z) 73 | 74 | (b.z + s * t.z) * a.x = (b.x + s * t.x) 75 | (b.z + s * t.z) * a.y = (b.y + s * t.y) 76 | 77 | (b.z + s * t.z) = (b.x + s * t.x) / a.x 78 | (b.z + s * t.z) = (b.y + s * t.y) / a.y 79 | 80 | (b.z + s * t.z) = (b.x + s * t.x) / a.x = (b.y + s * t.y) / a.y 81 | 82 | (b.x + s * t.x) / a.x = (b.y + s * t.y) / a.y 83 | 84 | a.y * (b.x + s * t.x) = a.x * (b.y + s * t.y) 85 | 86 | a.y * b.x + a.y * s * t.x = a.x * b.y + a.x * s * t.y 87 | 88 | a.y * b.x - a.x * b.y = a.x * s * t.y - a.y * s * t.x 89 | 90 | a.y * b.x - a.x * b.y = s * (a.x * t.y - a.y * t.x) 91 | 92 | s = (a.y * b.x - a.x * b.y) / (a.x * t.y - a.y * t.x) 93 | ``` 94 | 95 | Note that this process assumes nothing about the length of the translation vector `t`. It can be scaled regardless of its original length. Also note how, during the process of deriving `s`, `b.z` was eliminated. This is why the function does not take a `z` component. It is important to note that `b.xy` must come from a 3d point in camera space and not from a keypoint. The translation is what makes it unecessary to have the depth component. 96 | -------------------------------------------------------------------------------- /akaze/src/detector_response.rs: -------------------------------------------------------------------------------- 1 | use crate::{derivatives, evolution::EvolutionStep, image::GrayFloatImage, Akaze}; 2 | use ndarray::azip; 3 | 4 | #[cfg(feature = "rayon")] 5 | use rayon::prelude::*; 6 | 7 | impl Akaze { 8 | fn compute_multiscale_derivatives(&self, evolutions: &mut [EvolutionStep]) { 9 | let process_evolution = |evolution: &mut EvolutionStep| { 10 | // The image decreases in size by a factor which is 2^octave. 11 | let ratio = 2.0f64.powi(evolution.octave as i32); 12 | // The scale of the edge filter. 13 | let sigma_size = f64::round(evolution.esigma * self.derivative_factor / ratio) as u32; 14 | compute_multiscale_derivatives_for_evolution(evolution, sigma_size); 15 | }; 16 | #[cfg(not(feature = "rayon"))] 17 | for evolution in evolutions.iter_mut() { 18 | process_evolution(evolution); 19 | } 20 | #[cfg(feature = "rayon")] 21 | evolutions.into_par_iter().for_each(|evolution| { 22 | process_evolution(evolution); 23 | }); 24 | } 25 | 26 | /// Compute the detector response - the determinant of the Hessian - and save the result 27 | /// in the evolutions. 28 | /// 29 | /// # Arguments 30 | /// * `evolutions` - The computed evolutions. 31 | /// * `options` - The options 32 | #[allow(non_snake_case, clippy::suspicious_operation_groupings)] 33 | pub fn detector_response(&self, evolutions: &mut [EvolutionStep]) { 34 | self.compute_multiscale_derivatives(evolutions); 35 | let process_evolution = |evolution: &mut EvolutionStep| { 36 | let ratio = f64::powi(2.0, evolution.octave as i32); 37 | let sigma_size = f64::round(evolution.esigma * self.derivative_factor / ratio); 38 | let sigma_size_quat = sigma_size.powi(4) as f32; 39 | evolution.Ldet = GrayFloatImage::new(evolution.Lxx.width(), evolution.Lxx.height()); 40 | azip!(( 41 | Ldet in evolution.Ldet.mut_array2(), 42 | &Lxx in evolution.Lxx.ref_array2(), 43 | &Lyy in evolution.Lyy.ref_array2(), 44 | &Lxy in evolution.Lxy.ref_array2(), 45 | ) { 46 | *Ldet = (Lxx * Lyy - Lxy * Lxy) * sigma_size_quat; 47 | }); 48 | }; 49 | #[cfg(not(feature = "rayon"))] 50 | for evolution in evolutions.iter_mut() { 51 | process_evolution(evolution); 52 | } 53 | #[cfg(feature = "rayon")] 54 | evolutions.into_par_iter().for_each(|evolution| { 55 | process_evolution(evolution); 56 | }); 57 | } 58 | } 59 | 60 | fn compute_multiscale_derivatives_for_evolution(evolution: &mut EvolutionStep, sigma_size: u32) { 61 | #[cfg(not(feature = "rayon"))] 62 | { 63 | evolution.Lx = derivatives::scharr_horizontal(&evolution.Lsmooth, sigma_size); 64 | evolution.Ly = derivatives::scharr_vertical(&evolution.Lsmooth, sigma_size); 65 | evolution.Lxx = derivatives::scharr_horizontal(&evolution.Lx, sigma_size); 66 | evolution.Lyy = derivatives::scharr_vertical(&evolution.Ly, sigma_size); 67 | evolution.Lxy = derivatives::scharr_vertical(&evolution.Lx, sigma_size); 68 | } 69 | #[cfg(feature = "rayon")] 70 | { 71 | (evolution.Lx, evolution.Ly) = rayon::join( 72 | || derivatives::scharr_horizontal(&evolution.Lsmooth, sigma_size), 73 | || derivatives::scharr_vertical(&evolution.Lsmooth, sigma_size), 74 | ); 75 | (evolution.Lxx, (evolution.Lyy, evolution.Lxy)) = rayon::join( 76 | || derivatives::scharr_horizontal(&evolution.Lx, sigma_size), 77 | || { 78 | rayon::join( 79 | || derivatives::scharr_vertical(&evolution.Ly, sigma_size), 80 | || derivatives::scharr_vertical(&evolution.Lx, sigma_size), 81 | ) 82 | }, 83 | ) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /akaze/src/fed_tau.rs: -------------------------------------------------------------------------------- 1 | use std::f64::consts::PI; 2 | /// derived from C++ code by Pablo F. Alcantarilla, Jesus Nuevo in the 3 | /// AKAZE library. Notes from original author of the C++ code: 4 | /// 5 | /// This code is derived from FED/FJ library from Grewenig et al., 6 | /// The FED/FJ library allows solving more advanced problems 7 | /// Please look at the following papers for more information about FED: 8 | /// S. Grewenig, J. Weickert, C. Schroers, A. Bruhn. Cyclic Schemes for 9 | /// PDE-Based Image Analysis. Technical Report No. 327, Department of Mathematics, 10 | /// Saarland University, Saarbrücken, Germany, March 2013 11 | /// S. Grewenig, J. Weickert, A. Bruhn. From box filtering to fast explicit diffusion. 12 | /// DAGM, 2010 13 | /// 14 | /// This function allocates an array of the least number of time steps such 15 | /// that a certain stopping time for the whole process can be obtained and fills 16 | /// it with the respective FED time step sizes for one cycle 17 | /// 18 | /// # Arguments 19 | /// * `T` - Desired process stopping time 20 | /// * `M` - Desired number of cycles 21 | /// * `tau_max` - Stability limit for the explicit scheme 22 | /// * `reordering` - Reordering flag 23 | /// # Return value 24 | /// The vector with the dynamic step sizes 25 | #[allow(non_snake_case)] 26 | pub fn fed_tau_by_process_time(T: f64, M: i32, tau_max: f64, reordering: bool) -> Vec { 27 | // All cycles have the same fraction of the stopping time 28 | fed_tau_by_cycle_time(T / f64::from(M), tau_max, reordering) 29 | } 30 | 31 | /// This function allocates an array of the least number of time steps such 32 | /// that a certain stopping time for the whole process can be obtained and fills it 33 | /// it with the respective FED time step sizes for one cycle 34 | /// 35 | /// # Arguments 36 | /// * `t` - Desired cycle stopping time 37 | /// * `tau_max` - Stability limit for the explicit scheme 38 | /// * `reordering` - Reordering flag 39 | /// # Return value 40 | /// tau The vector with the dynamic step sizes 41 | #[allow(non_snake_case)] 42 | fn fed_tau_by_cycle_time(t: f64, tau_max: f64, reordering: bool) -> Vec { 43 | // number of time steps 44 | let n = (f64::ceil(f64::sqrt(3.0 * t / tau_max + 0.25) - 0.5f64 - 1.0e-8) + 0.5) as usize; 45 | // Ratio of t we search to maximal t 46 | let scale = 3.0 * t / (tau_max * ((n * (n + 1)) as f64)); 47 | fed_tau_internal(n, scale, tau_max, reordering) 48 | } 49 | 50 | /// This function allocates an array of time steps and fills it with FED 51 | /// time step sizes 52 | /// 53 | /// # Arguments 54 | /// * `n` - Number of internal steps 55 | /// * `scale` - Ratio of t we search to maximal t 56 | /// * `tau_max` - Stability limit for the explicit scheme 57 | /// * `reordering` - Reordering flag 58 | /// # Return value 59 | /// The vector with the dynamic step sizes 60 | fn fed_tau_internal(n: usize, scale: f64, tau_max: f64, reordering: bool) -> Vec { 61 | let tau: Vec = (0..n) 62 | .map(|k| { 63 | let c: f64 = 1.0f64 / (4.0f64 * (n as f64) + 2.0f64); 64 | let d: f64 = scale * tau_max / 2.0f64; 65 | let h = f64::cos(PI * (2.0f64 * (k as f64) + 1.0f64) * c); 66 | d / (h * h) 67 | }) 68 | .collect(); 69 | if reordering { 70 | // Permute list of time steps according to chosen reordering function 71 | // Choose kappa cycle with k = n/2 72 | // This is a heuristic. We can use Leja ordering instead! 73 | let kappa = n / 2; 74 | let mut prime = n + 1; 75 | while !primal::is_prime(prime as u64) { 76 | prime += 1; 77 | } 78 | let mut k = 0; 79 | (0..n) 80 | .map(move |_| { 81 | let mut index = ((k + 1) * kappa) % prime - 1; 82 | while index >= n { 83 | k += 1; 84 | index = ((k + 1) * kappa) % prime - 1; 85 | } 86 | k += 1; 87 | tau[index] 88 | }) 89 | .collect() 90 | } else { 91 | tau 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /cv-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! # Rust CV Core 2 | //! 3 | //! This library provides common abstractions and types for computer vision (CV) in Rust. 4 | //! All the crates in the rust-cv ecosystem that have or depend on CV types depend on this crate. 5 | //! This includes things like camera model traits, bearings, poses, keypoints, etc. The crate is designed to 6 | //! be very small so that it adds negligable build time. It pulls in some dependencies 7 | //! that will probably be brought in by writing computer vision code normally. 8 | //! The core concept is that all CV crates can work together with each other by using the 9 | //! abstractions and types specified in this crate. 10 | //! 11 | //! The crate is designed to work with `#![no_std]`, even without an allocator. `libm` is used 12 | //! (indirectly through [`num-traits`]) for all math algorithms that aren't present in `std`. Any 13 | //! code that doesn't need to be shared across all CV crates should not belong in this repository. 14 | //! If there is a good reason to put code that some crates may need into `cv-core`, it should be 15 | //! gated behind a feature. 16 | //! 17 | //! ## Triangulation 18 | //! 19 | //! Several of the traits with in `cv-core`, such as [`TriangulatorObservations`], must perform a process 20 | //! called [triangulation](https://en.wikipedia.org/wiki/Triangulation). In computer vision, this problem 21 | //! occurs quite often, as we often have some of the following data: 22 | //! 23 | //! * [The pose of a camera](WorldToCamera) 24 | //! * [The relative pose of a camera](CameraToCamera) 25 | //! * [A bearing direction at which lies a feature](nalgebra::UnitVector3) 26 | //! 27 | //! We have to take this data and produce a 3d point. Cameras have an optical center which all bearings protrude from. 28 | //! This is often refered to as the focal point in a standard camera, but in computer vision the term optical center 29 | //! is prefered, as it is a generalized concept. What typically happens in triangulation is that we have (at least) 30 | //! two optical centers and a bearing (direction) out of each of those optical centers approximately pointing towards 31 | //! the 3d point. In an ideal world, these bearings would point exactly at the point and triangulation would be achieved 32 | //! simply by solving the equation for the point of intersection. Unfortunately, the real world throws a wrench at us, as 33 | //! the bearings wont actually intersect since they are based on noisy data. This is what causes us to need different 34 | //! triangulation algorithms, which deal with the error in different ways and have different characteristics. 35 | //! 36 | //! Here is an example where we have two pinhole cameras A and B. The `@` are used to show the 37 | //! [virtual image plane](https://en.wikipedia.org/wiki/Pinhole_camera_model). The virtual image plane can be thought 38 | //! of as a surface in front of the camera through which the light passes through from the point to the optical center `O`. 39 | //! The points `a` and `b` are normalized image coordinates which describe the position on the virtual image plane which 40 | //! the light passed through from the point to the optical center on cameras `A` and `B` respectively. We know the 41 | //! exact pose (position and orientation) of each of these two cameras, and we also know the normalized image coordinates, 42 | //! which we can use to compute a bearing. We are trying to solve for the point `p` which would cause the ray of light to 43 | //! pass through points `a` and `b` followed by `O`. 44 | //! 45 | //! - `p` the point we are trying to triangulate 46 | //! - `a` the normalized keypoint on camera A 47 | //! - `b` the normalized keypoint on camera B 48 | //! - `O` the optical center of a camera 49 | //! - `@` the virtual image plane 50 | //! 51 | //! ```text 52 | //! @ 53 | //! @ 54 | //! p--------b--------O 55 | //! / @ 56 | //! / @ 57 | //! / @ 58 | //! / @ 59 | //! @@@@@@@a@@@@@ 60 | //! / 61 | //! / 62 | //! / 63 | //! O 64 | //! ``` 65 | 66 | #![no_std] 67 | 68 | mod camera; 69 | mod keypoint; 70 | mod matches; 71 | mod point; 72 | mod pose; 73 | mod so3; 74 | mod triangulation; 75 | 76 | pub use camera::*; 77 | pub use keypoint::*; 78 | pub use matches::*; 79 | pub use nalgebra; 80 | pub use point::*; 81 | pub use pose::*; 82 | pub use sample_consensus; 83 | pub use so3::*; 84 | pub use triangulation::*; 85 | -------------------------------------------------------------------------------- /nister-stewenius/tests/manual.rs: -------------------------------------------------------------------------------- 1 | // use cv_core::nalgebra::{IsometryMatrix3, Rotation3, Vector2, Vector3}; 2 | // use cv_core::sample_consensus::{Estimator, Model}; 3 | // use cv_core::{CameraPoint, CameraToCamera, FeatureMatch, Pose}; 4 | // use cv_pinhole::NormalizedKeyPoint; 5 | 6 | // const SAMPLE_POINTS: usize = 16; 7 | // const RESIDUAL_THRESHOLD: f64 = 1e-4; 8 | 9 | // const ROT_MAGNITUDE: f64 = 0.2; 10 | // const POINT_BOX_SIZE: f64 = 2.0; 11 | // const POINT_DISTANCE: f64 = 3.0; 12 | 13 | // #[test] 14 | // fn randomized() { 15 | // let successes = (0..1000).filter(|_| run_round()).count(); 16 | // eprintln!("successes: {}", successes); 17 | // assert!(successes > 950); 18 | // } 19 | 20 | // fn run_round() -> bool { 21 | // let mut success = true; 22 | // let (real_pose, aps, bps) = some_test_data(); 23 | // let matches = aps.iter().zip(&bps).map(|(&a, &b)| FeatureMatch(a, b)); 24 | // let eight_point = eight_point::EightPoint::new(); 25 | // let essential = eight_point 26 | // .estimate(matches.clone()) 27 | // .expect("didn't get any essential matrix"); 28 | // for m in matches.clone() { 29 | // if essential.residual(&m).abs() > RESIDUAL_THRESHOLD { 30 | // success = false; 31 | // eprintln!("failed residual check: {}", essential.residual(&m).abs()); 32 | // } 33 | // } 34 | 35 | // // Get the possible poses for the essential matrix created from `pose`. 36 | // let estimate_pose = match essential.pose_solver().solve_unscaled(matches) { 37 | // Some(pose) => pose, 38 | // None => { 39 | // return false; 40 | // } 41 | // }; 42 | 43 | // let rot_axis_residual = 1.0 44 | // - estimate_pose 45 | // .0 46 | // .rotation 47 | // .axis() 48 | // .unwrap() 49 | // .dot(&real_pose.0.rotation.axis().unwrap()); 50 | // let rot_angle_residual = 51 | // (estimate_pose.0.rotation.angle() - real_pose.0.rotation.angle()).abs(); 52 | // let translation_residual = 1.0 53 | // - real_pose 54 | // .0 55 | // .translation 56 | // .vector 57 | // .normalize() 58 | // .dot(&estimate_pose.0.translation.vector.normalize()); 59 | // success &= rot_axis_residual < RESIDUAL_THRESHOLD; 60 | // success &= rot_angle_residual < RESIDUAL_THRESHOLD; 61 | // success &= translation_residual < RESIDUAL_THRESHOLD; 62 | // if !success { 63 | // eprintln!("rot angle residual({})", rot_angle_residual); 64 | // eprintln!("rot axis residual({})", rot_axis_residual); 65 | // eprintln!("translation residual({})", translation_residual); 66 | // eprintln!("real pose: {:?}", real_pose); 67 | // eprintln!("estimate pose: {:?}", estimate_pose); 68 | // } 69 | // success 70 | // } 71 | 72 | // /// Gets a random relative pose, input points A, input points B, and A point depths. 73 | // fn some_test_data() -> ( 74 | // CameraToCamera, 75 | // [NormalizedKeyPoint; SAMPLE_POINTS], 76 | // [NormalizedKeyPoint; SAMPLE_POINTS], 77 | // ) { 78 | // // The relative pose orientation is fixed and translation is random. 79 | // let relative_pose = CameraToCamera(IsometryMatrix3::from_parts( 80 | // Vector3::new_random().into(), 81 | // Rotation3::new(Vector3::new_random() * std::f64::consts::PI * 2.0 * ROT_MAGNITUDE), 82 | // )); 83 | 84 | // // Generate A's camera points. 85 | // let cams_a = (0..SAMPLE_POINTS) 86 | // .map(|_| { 87 | // let mut a = Vector3::new_random() * POINT_BOX_SIZE; 88 | // a.x -= 0.5 * POINT_BOX_SIZE; 89 | // a.y -= 0.5 * POINT_BOX_SIZE; 90 | // a.z += POINT_DISTANCE; 91 | // CameraPoint(a.push(1.0)) 92 | // }) 93 | // .collect::>() 94 | // .into_iter(); 95 | 96 | // // Generate B's camera points. 97 | // let cams_b = cams_a.clone().map(|a| relative_pose.transform(a)); 98 | 99 | // let mut kps_a = [NormalizedKeyPoint(Vector2::zeros().into()); SAMPLE_POINTS]; 100 | // for (keypoint, camera) in kps_a.iter_mut().zip(cams_a) { 101 | // *keypoint = NormalizedKeyPoint::from_camera_point(camera).unwrap(); 102 | // } 103 | // let mut kps_b = [NormalizedKeyPoint(Vector2::zeros().into()); SAMPLE_POINTS]; 104 | // for (keypoint, camera) in kps_b.iter_mut().zip(cams_b.clone()) { 105 | // *keypoint = NormalizedKeyPoint::from_camera_point(camera).unwrap(); 106 | // } 107 | 108 | // (relative_pose, kps_a, kps_b) 109 | // } 110 | -------------------------------------------------------------------------------- /cv-optimize/src/single_view_optimizer.rs: -------------------------------------------------------------------------------- 1 | use cv_core::{FeatureWorldMatch, Pose, Projective, Se3TangentSpace, WorldToCamera}; 2 | use cv_geom::epipolar; 3 | 4 | pub(crate) fn landmark_delta( 5 | pose: WorldToCamera, 6 | landmark: FeatureWorldMatch, 7 | ) -> Option { 8 | let FeatureWorldMatch(bearing, world_point) = landmark; 9 | let camera_point = pose.transform(world_point); 10 | Some(epipolar::world_pose_gradient( 11 | camera_point.point()?.coords, 12 | bearing, 13 | )) 14 | } 15 | 16 | pub fn single_view_simple_optimize_l1( 17 | mut pose: WorldToCamera, 18 | epsilon: f64, 19 | optimization_rate: f64, 20 | iterations: usize, 21 | landmarks: &[FeatureWorldMatch], 22 | ) -> WorldToCamera { 23 | if landmarks.is_empty() { 24 | return pose; 25 | } 26 | let mut best_trans = f64::INFINITY; 27 | let mut best_rot = f64::INFINITY; 28 | let mut no_improve_for = 0; 29 | for iteration in 0..iterations { 30 | let tscale = pose.isometry().translation.vector.norm(); 31 | let mut l1sum = Se3TangentSpace::identity(); 32 | let mut ts = 0.0; 33 | let mut rs = 0.0; 34 | for &landmark in landmarks { 35 | if let Some(tangent) = landmark_delta(pose, landmark) { 36 | ts += (tangent.translation.norm() + tscale * epsilon).recip(); 37 | rs += (tangent.rotation.norm() + epsilon).recip(); 38 | l1sum += tangent.l1(); 39 | } 40 | } 41 | 42 | let delta = l1sum 43 | .scale(optimization_rate) 44 | .scale_translation(ts.recip()) 45 | .scale_rotation(rs.recip()); 46 | 47 | no_improve_for += 1; 48 | let t = l1sum.translation.norm(); 49 | let r = l1sum.rotation.norm(); 50 | if best_trans > t { 51 | best_trans = t; 52 | no_improve_for = 0; 53 | } 54 | if best_rot > r { 55 | best_rot = r; 56 | no_improve_for = 0; 57 | } 58 | 59 | if no_improve_for >= 50 { 60 | log::info!( 61 | "terminating single-view optimization due to stabilizing on iteration {}", 62 | iteration 63 | ); 64 | log::info!("tangent rotation magnitude: {}", l1sum.rotation.norm()); 65 | break; 66 | } 67 | 68 | // Update the pose. 69 | pose = (delta.isometry() * pose.isometry()).into(); 70 | 71 | if iteration == iterations - 1 { 72 | log::info!("terminating single-view optimization due to reaching maximum iterations"); 73 | log::info!("tangent rotation magnitude: {}", l1sum.rotation.norm()); 74 | break; 75 | } 76 | } 77 | pose 78 | } 79 | 80 | pub fn single_view_simple_optimize_l2( 81 | mut pose: WorldToCamera, 82 | optimization_rate: f64, 83 | iterations: usize, 84 | landmarks: &[FeatureWorldMatch], 85 | ) -> WorldToCamera { 86 | if landmarks.is_empty() { 87 | return pose; 88 | } 89 | let mut best_trans = f64::INFINITY; 90 | let mut best_rot = f64::INFINITY; 91 | let mut no_improve_for = 0; 92 | let inv_landmark_len = (landmarks.len() as f64).recip(); 93 | for iteration in 0..iterations { 94 | let mut l2sum = Se3TangentSpace::identity(); 95 | for &landmark in landmarks { 96 | if let Some(tangent) = landmark_delta(pose, landmark) { 97 | l2sum += tangent; 98 | } 99 | } 100 | 101 | let tangent = l2sum.scale(inv_landmark_len); 102 | let delta = tangent.scale(optimization_rate); 103 | 104 | no_improve_for += 1; 105 | let t = l2sum.translation.norm(); 106 | let r = l2sum.rotation.norm(); 107 | if best_trans > t { 108 | best_trans = t; 109 | no_improve_for = 0; 110 | } 111 | if best_rot > r { 112 | best_rot = r; 113 | no_improve_for = 0; 114 | } 115 | 116 | if no_improve_for >= 50 { 117 | log::info!( 118 | "terminating single-view optimization due to stabilizing on iteration {}", 119 | iteration 120 | ); 121 | log::info!("tangent rotation magnitude: {}", tangent.rotation.norm()); 122 | break; 123 | } 124 | 125 | // Update the pose. 126 | pose = (delta.isometry() * pose.isometry()).into(); 127 | 128 | if iteration == iterations - 1 { 129 | log::info!("terminating single-view optimization due to reaching maximum iterations"); 130 | log::info!("tangent rotation magnitude: {}", tangent.rotation.norm()); 131 | break; 132 | } 133 | } 134 | pose 135 | } 136 | -------------------------------------------------------------------------------- /tutorial/src/chapter2-first-program.md: -------------------------------------------------------------------------------- 1 | # First program 2 | 3 | In this chapter, we will be reviewing our first program. It is not even really related to computer vision, but it will help you get started with some basic tools that will be used again later and get familiar with the tutorial process. We will just be loading an image, drawing random points on it, and displaying it. This is very basic, but at least you can make sure you have everything set up correctly before attacking more ambitious problems. 4 | 5 | ## Running the program 6 | 7 | In order to start, we will clone the [Rust CV mono-repo](https://github.com/rust-cv/cv). Change into the repository directory. Run the following: 8 | 9 | ```bash 10 | cargo run --release --bin chapter2-first-program 11 | ``` 12 | 13 | You should see a grayscale image (from the Kitti dataset) with thousands of small blue translucent crosses drawn on it. If you see this, then everything is working on your computer. Here is what it should look like: 14 | 15 | ![Random points](https://rust-cv.github.io/res/tutorial-images/random-points.png) 16 | 17 | We are now going to go through the code in `tutorial-code/chapter2-first-program/src/main.rs` piece-by-piece. We will do this for each chapter of the book and its relevant example. It is recommended to tweak the code in every chapter to get a better idea of how the code works. All of the code samples can be found in `tutorial-code`. We will skip talking about the imports unless they are relevant at any point. Comments will also be omitted since we will be discussing the code in this tutorial. 18 | 19 | ## The code 20 | 21 | ### Load the image 22 | 23 | ```rust 24 | let src_image = image::open("res/0000000000.png").expect("failed to open image file"); 25 | ``` 26 | 27 | This code will load an image called `res/0000000000.png` relative to the current directory you are running this program from. This will only work when you are in the root of the Rust CV mono-repo, which is where the `res` directory is located. 28 | 29 | ### Create a random number generator 30 | 31 | ```rust 32 | let mut rng = rand::thread_rng(); 33 | ``` 34 | 35 | This code creates a random number generator (RNG) using the `rand` crate. This random number generator will use entropy information from the OS to seed a fairly robust PRNG on a regular basis, and it is used here because it is very simple to create one. 36 | 37 | ### Drawing the crosses 38 | 39 | ```rust 40 | let mut image_canvas = drawing::Blend(src_image.to_rgba8()); 41 | for _ in 0..5000 { 42 | let x = rng.gen_range(0..src_image.width()) as i32; 43 | let y = rng.gen_range(0..src_image.height()) as i32; 44 | drawing::draw_cross_mut(&mut image_canvas, Rgba([0, 255, 255, 128]), x, y); 45 | } 46 | ``` 47 | 48 | This section of code is going to iterate `5000` times. Each iteration it is going to generate a random x and y position that is on the image. We then use the `imageproc::drawing` module to draw a cross on the spot in question. Note that the `image_canvas` is created by making an RGBA version of the original grayscale image and then wrapping it in the `imageproc::drawing::Blend` adapter. This is done so that when we draw the cross onto the canvas that it will use the alpha value (which we set to `128`) to make the cross translucent. This is useful so that we can see through the cross a little bit so that it doesn't totally obscure the underlying image. 49 | 50 | ### Changing the image back into DynamicImage 51 | 52 | ```rust 53 | let out_image = DynamicImage::ImageRgba8(image_canvas.0); 54 | ``` 55 | 56 | We now take the `RgbaImage` and turn it into a `DynamicImage`. This is done because `DynamicImage` is a wrapper around all image types that has convenient save and load methods, and we actually used it when we originally loaded the image. 57 | 58 | ### Write the image to a temporary file 59 | 60 | ```rust 61 | let image_file_path = tempfile::Builder::new() 62 | .suffix(".png") 63 | .tempfile() 64 | .unwrap() 65 | .into_temp_path(); 66 | out_image.save(&image_file_path).unwrap(); 67 | ``` 68 | 69 | Here we use the `tempfile` crate to create a temporary file. The benefit of a temporary file is that it can be deleted automatically for us when we are done with it. In this case it may not get deleted automatically because the OS image viewer will later be used and it may prevent the file from being deleted, but it is good practice to create temporary files to store image. 70 | 71 | After we create the temporary file path, we write to the path by saving the output image to it. 72 | 73 | ### Open the image 74 | 75 | ```rust 76 | open::that(&image_file_path).unwrap(); 77 | std::thread::sleep(std::time::Duration::from_secs(5)); 78 | ``` 79 | 80 | We use the `open` crate here to open the image file. This will automatically use the program configured on your computer for viewing images to open the image. Since the image program does not open the file right away, we have to sleep for some period of time to ensure we don't delete the temporary file. 81 | 82 | ## End 83 | 84 | This is the end of this chapter. -------------------------------------------------------------------------------- /cv-sfm/src/export.rs: -------------------------------------------------------------------------------- 1 | use cv_core::nalgebra::{Point3, Vector3}; 2 | use ply_rs::{ 3 | ply::{ 4 | Addable, DefaultElement, ElementDef, Encoding, Ply, Property, PropertyDef, PropertyType, 5 | ScalarType, 6 | }, 7 | writer::Writer, 8 | }; 9 | use std::io::Write; 10 | 11 | const CAMERA_COLOR: [u8; 3] = [255, 0, 255]; 12 | 13 | pub struct ExportCamera { 14 | pub optical_center: Point3, 15 | pub up_direction: Vector3, 16 | pub forward_direction: Vector3, 17 | pub focal_length: f64, 18 | } 19 | 20 | pub fn export( 21 | mut writer: impl Write, 22 | points_and_colors: Vec<(Point3, [u8; 3])>, 23 | cameras: Vec, 24 | camera_faces: bool, 25 | ) { 26 | // crete a ply objet 27 | let mut ply = Ply::::new(); 28 | ply.header.encoding = Encoding::Ascii; 29 | ply.header 30 | .comments 31 | .push("Exported from rust-cv/vslam-sandbox".to_string()); 32 | 33 | // Define the vertex element, which will be used for face verticies and reconstruction points. 34 | let mut point_element = ElementDef::new("vertex".to_string()); 35 | let p = PropertyDef::new("x".to_string(), PropertyType::Scalar(ScalarType::Double)); 36 | point_element.properties.add(p); 37 | let p = PropertyDef::new("y".to_string(), PropertyType::Scalar(ScalarType::Double)); 38 | point_element.properties.add(p); 39 | let p = PropertyDef::new("z".to_string(), PropertyType::Scalar(ScalarType::Double)); 40 | point_element.properties.add(p); 41 | let p = PropertyDef::new("red".to_string(), PropertyType::Scalar(ScalarType::UChar)); 42 | point_element.properties.add(p); 43 | let p = PropertyDef::new("green".to_string(), PropertyType::Scalar(ScalarType::UChar)); 44 | point_element.properties.add(p); 45 | let p = PropertyDef::new("blue".to_string(), PropertyType::Scalar(ScalarType::UChar)); 46 | point_element.properties.add(p); 47 | ply.header.elements.add(point_element); 48 | 49 | if camera_faces { 50 | // Define the face element which will be used for cameras. 51 | let mut face_element = ElementDef::new("face".to_string()); 52 | let vertex_list = PropertyDef::new( 53 | "vertex_index".to_string(), 54 | PropertyType::List(ScalarType::UChar, ScalarType::Int), 55 | ); 56 | face_element.properties.add(vertex_list); 57 | ply.header.elements.add(face_element); 58 | } 59 | 60 | let mut faces: Vec = vec![]; 61 | let mut vertices: Vec = vec![]; 62 | 63 | let mut add_vertex = |p: Point3, [r, g, b]: [u8; 3]| -> usize { 64 | let pos = vertices.len(); 65 | let mut point = DefaultElement::new(); 66 | point.insert("x".to_string(), Property::Double(p.x)); 67 | point.insert("y".to_string(), Property::Double(p.y)); 68 | point.insert("z".to_string(), Property::Double(p.z)); 69 | point.insert("red".to_string(), Property::UChar(r)); 70 | point.insert("green".to_string(), Property::UChar(g)); 71 | point.insert("blue".to_string(), Property::UChar(b)); 72 | vertices.push(point); 73 | pos 74 | }; 75 | 76 | let mut add_triangle = |a: usize, b: usize, c: usize| -> usize { 77 | let pos = faces.len(); 78 | let mut face = DefaultElement::new(); 79 | face.insert( 80 | "vertex_index".to_string(), 81 | Property::ListInt(vec![a as i32, b as i32, c as i32]), 82 | ); 83 | faces.push(face); 84 | pos 85 | }; 86 | 87 | // Add cameras 88 | for ExportCamera { 89 | optical_center, 90 | up_direction, 91 | forward_direction, 92 | focal_length, 93 | } in cameras 94 | { 95 | let right_direction = forward_direction.cross(&up_direction); 96 | let center_point = add_vertex(optical_center, CAMERA_COLOR); 97 | let [up_right, up_left, down_left, down_right] = 98 | [(1, 1), (1, -1), (-1, -1), (-1, 1)].map(|(up, right)| { 99 | add_vertex( 100 | optical_center 101 | + forward_direction * focal_length 102 | + up as f64 * up_direction * focal_length 103 | + right as f64 * right_direction * focal_length, 104 | CAMERA_COLOR, 105 | ) 106 | }); 107 | 108 | if camera_faces { 109 | add_triangle(center_point, down_right, up_right); 110 | add_triangle(center_point, up_right, up_left); 111 | add_triangle(center_point, up_left, down_left); 112 | add_triangle(center_point, down_left, down_right); 113 | } 114 | } 115 | 116 | // Add points 117 | for (p, c) in points_and_colors { 118 | add_vertex(p, c); 119 | } 120 | 121 | ply.payload.insert("vertex".to_string(), vertices); 122 | if camera_faces { 123 | ply.payload.insert("face".to_string(), faces); 124 | } 125 | 126 | // set up a writer 127 | let w = Writer::new(); 128 | w.write_ply(&mut writer, &mut ply).unwrap(); 129 | } 130 | -------------------------------------------------------------------------------- /cv/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! # `cv` 2 | //! 3 | //! Batteries-included pure-Rust computer vision crate 4 | //! 5 | //! This crate should only be used for documentation/reference and for quickly creating and 6 | //! running a computer vision sample/routine. It is useful for tutorials and also for 7 | //! experts who want to run something once as a script. It also stores all of the 8 | //! things useful for computer vision in the Rust ecosystem in one place for 9 | //! discoverability. If you are making a production application, import the dependencies 10 | //! from this crate individually so that you don't have an explosive number of dependencies. 11 | //! Although not recommended, you can also disable default features on this crate and 12 | //! enable specific features on this crate just to get the functionality you want. 13 | //! 14 | //! All of the basic computer vision types/dependencies are included in the root of the crate. 15 | //! Modules are created to store algorithms and data structures which may or may not be used. 16 | //! Almost all of the things in these modules come from optional libraries. 17 | //! These modules comprise the core functionality required to perform computer vision tasks. 18 | //! 19 | //! ## Modules 20 | //! * [`camera`] - camera models to convert image coordinates into bearings (and back) 21 | //! * [`consensus`] - finding the best estimated model from noisy data 22 | //! * [`geom`] - computational geometry algorithms used in computer vision 23 | //! * [`estimate`] - estimation of models from data 24 | //! * [`feature`] - feature extraction and description 25 | //! * [`image`] - image opening and processing/manipulation 26 | //! * [`knn`] - searching for nearest neighbors in small or large datasets 27 | //! * [`optimize`] - optimizing models to fit data 28 | //! * [`mvg`] - multiple-view geometry (visual odometry, SfM, vSLAM) 29 | //! * [`video`] - video opening and camera capture 30 | //! * [`vis`] - visualization 31 | 32 | #![no_std] 33 | 34 | pub use cv_core::{sample_consensus::*, *}; 35 | 36 | #[cfg(feature = "space")] 37 | pub use space::Metric; 38 | 39 | #[cfg(feature = "bitarray")] 40 | pub use bitarray; 41 | 42 | /// Camera models (see [`video`] for camera capture) 43 | pub mod camera { 44 | /// The pinhole camera model 45 | #[cfg(feature = "cv-pinhole")] 46 | pub use cv_pinhole as pinhole; 47 | } 48 | 49 | /// Consensus algorithms (RANSAC) 50 | pub mod consensus { 51 | #[cfg(feature = "arrsac")] 52 | pub use arrsac::Arrsac; 53 | } 54 | 55 | /// Computational geometry 56 | pub mod geom { 57 | #[cfg(feature = "cv-geom")] 58 | pub use cv_geom::*; 59 | } 60 | 61 | /// Estimation algorithms 62 | pub mod estimate { 63 | #[cfg(feature = "eight-point")] 64 | pub use eight_point::EightPoint; 65 | #[cfg(feature = "lambda-twist")] 66 | pub use lambda_twist::LambdaTwist; 67 | #[cfg(feature = "nister-stewenius")] 68 | pub use nister_stewenius::NisterStewenius; 69 | } 70 | 71 | /// Feature detection and description 72 | pub mod feature { 73 | /// A robust and fast feature detector 74 | #[cfg(feature = "akaze")] 75 | pub mod akaze { 76 | pub use akaze::*; 77 | } 78 | } 79 | 80 | /// Image opening and processing/manipulation 81 | pub mod image { 82 | /// Re-export of [`image`] to open and save images 83 | #[cfg(feature = "image")] 84 | #[allow(clippy::module_inception)] 85 | pub mod image { 86 | pub use image::*; 87 | } 88 | 89 | /// Re-export of [`imageproc`] crate for image manipulation routines 90 | #[cfg(feature = "imageproc")] 91 | pub mod imageproc { 92 | pub use imageproc::*; 93 | } 94 | 95 | /// Re-export of [`ndarray-vision`] for image manipulation routines 96 | #[cfg(feature = "ndarray-vision")] 97 | pub mod ndarray_vision { 98 | pub use ndarray_vision::*; 99 | } 100 | } 101 | 102 | /// Algorithms for performing k-NN searches 103 | pub mod knn { 104 | /// Re-export of [`hgg`] crate, an approximate nearest neighbor search map 105 | #[cfg(feature = "hgg")] 106 | pub mod hgg { 107 | pub use hgg::*; 108 | } 109 | 110 | /// Re-export of [`hnsw`] crate, an approximate nearest neighbor index search data structure 111 | #[cfg(feature = "hnsw")] 112 | pub mod hnsw { 113 | pub use hnsw::*; 114 | } 115 | 116 | #[cfg(all(feature = "space", feature = "alloc"))] 117 | pub use space::{KnnInsert, KnnMap, KnnPoints, LinearKnn}; 118 | 119 | #[cfg(feature = "space")] 120 | pub use space::{Knn, Metric, Neighbor}; 121 | } 122 | 123 | /// Optimization algorithms 124 | pub mod optimize { 125 | /// Levenberg-Marquardt 126 | #[cfg(feature = "levenberg-marquardt")] 127 | pub mod lm { 128 | pub use levenberg_marquardt::*; 129 | } 130 | } 131 | 132 | /// Multiple-view geometry (visual odometry, SfM, vSLAM) 133 | pub mod mvg { 134 | #[cfg(feature = "cv-sfm")] 135 | pub use cv_sfm as sfm; 136 | } 137 | 138 | /// Video and camera capture 139 | pub mod video { 140 | /// Re-export of [`eye`] crate, used for capturing camera input 141 | #[cfg(feature = "eye")] 142 | pub mod eye { 143 | pub use eye::*; 144 | } 145 | } 146 | 147 | /// Visualization utilities 148 | pub mod vis {} 149 | -------------------------------------------------------------------------------- /akaze/src/evolution.rs: -------------------------------------------------------------------------------- 1 | use crate::{fed_tau, Akaze, GrayFloatImage}; 2 | use log::*; 3 | 4 | #[derive(Debug)] 5 | #[allow(non_snake_case)] 6 | pub struct EvolutionStep { 7 | /// Evolution time 8 | pub etime: f64, 9 | /// Evolution sigma. For linear diffusion t = sigma^2 / 2 10 | pub esigma: f64, 11 | /// Image octave 12 | pub octave: u32, 13 | /// Image sublevel in each octave 14 | pub sublevel: u32, 15 | /// Integer sigma. For computing the feature detector responses 16 | pub sigma_size: u32, 17 | /// Evolution image 18 | pub Lt: GrayFloatImage, 19 | /// Smoothed image 20 | pub Lsmooth: GrayFloatImage, 21 | /// First order spatial derivative 22 | pub Lx: GrayFloatImage, 23 | /// First order spatial derivatives 24 | pub Ly: GrayFloatImage, 25 | /// Second order spatial derivative 26 | pub Lxx: GrayFloatImage, 27 | /// Second order spatial derivatives 28 | pub Lyy: GrayFloatImage, 29 | /// Second order spatial derivatives 30 | pub Lxy: GrayFloatImage, 31 | /// Diffusivity image 32 | pub Lflow: GrayFloatImage, 33 | /// Detector response 34 | pub Ldet: GrayFloatImage, 35 | /// fed_tau steps 36 | pub fed_tau_steps: Vec, 37 | } 38 | 39 | impl EvolutionStep { 40 | /// Construct a new EvolutionStep for a given octave and sublevel 41 | /// 42 | /// # Arguments 43 | /// * `octave` - The target octave. 44 | /// * `octave` - The target sublevel. 45 | /// * `options` - The options to use. 46 | fn new(octave: u32, sublevel: u32, options: &Akaze) -> EvolutionStep { 47 | let esigma = options.base_scale_offset 48 | * f64::powf( 49 | 2.0f64, 50 | f64::from(sublevel) / f64::from(options.num_sublevels) + f64::from(octave), 51 | ); 52 | let etime = 0.5 * (esigma * esigma); 53 | EvolutionStep { 54 | etime, 55 | esigma, 56 | octave, 57 | sublevel, 58 | sigma_size: esigma.round() as u32, 59 | Lt: GrayFloatImage::new(0, 0), 60 | Lsmooth: GrayFloatImage::new(0, 0), 61 | Lx: GrayFloatImage::new(0, 0), 62 | Ly: GrayFloatImage::new(0, 0), 63 | Lxx: GrayFloatImage::new(0, 0), 64 | Lyy: GrayFloatImage::new(0, 0), 65 | Lxy: GrayFloatImage::new(0, 0), 66 | Lflow: GrayFloatImage::new(0, 0), 67 | Ldet: GrayFloatImage::new(0, 0), 68 | fed_tau_steps: vec![], 69 | } 70 | } 71 | } 72 | 73 | impl Akaze { 74 | /// Allocate and calculate prerequisites to the construction of a scale space. 75 | /// 76 | /// # Arguments 77 | /// `width` - The width of the input image. 78 | /// `height` - The height of the input image. 79 | /// `options` - The configuration to use. 80 | pub fn allocate_evolutions(&self, width: u32, height: u32) -> Vec { 81 | let mut evolutions: Vec = (0..self.max_octave_evolution) 82 | .filter_map(|octave| { 83 | let rfactor = 2.0f64.powi(-(octave as i32)); 84 | let level_height = (f64::from(height) * rfactor) as u32; 85 | let level_width = (f64::from(width) * rfactor) as u32; 86 | let smallest_dim = std::cmp::min(level_width, level_height); 87 | // If the smallest dim is less than 40, terminate as we cannot detect features 88 | // at a scale that small. 89 | if smallest_dim < 40 { 90 | None 91 | } else { 92 | // At a smallest dimension size between 80, only include one sublevel, 93 | // as the amount of information in the image is limited. 94 | let sublevels = if smallest_dim < 80 { 95 | 1 96 | } else { 97 | self.num_sublevels 98 | }; 99 | // Return the sublevels. 100 | Some( 101 | (0..sublevels) 102 | .map(move |sublevel| EvolutionStep::new(octave, sublevel, self)), 103 | ) 104 | } 105 | }) 106 | .flatten() 107 | .collect(); 108 | // We need to set the tau steps. 109 | // This is used to produce each evolution. 110 | // Each tau corresponds to one diffusion time step. 111 | // In FED (Fast Explicit Diffusion) earlier time steps are smaller 112 | // because they are more unstable. Once it becomes more stable, the time 113 | // steps become larger. 114 | for i in 1..evolutions.len() { 115 | // Comute the total difference in time between evolutions. 116 | let ttime = evolutions[i].etime - evolutions[i - 1].etime; 117 | // Compute the separate tau steps and assign it to the evolution. 118 | evolutions[i].fed_tau_steps = fed_tau::fed_tau_by_process_time(ttime, 1, 0.25, true); 119 | debug!( 120 | "{} steps in evolution {}.", 121 | evolutions[i].fed_tau_steps.len(), 122 | i 123 | ); 124 | } 125 | evolutions 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /lambda-twist/tests/consensus.rs: -------------------------------------------------------------------------------- 1 | use approx::assert_relative_eq; 2 | use arrayvec::ArrayVec; 3 | use arrsac::Arrsac; 4 | use cv_core::{ 5 | nalgebra::{IsometryMatrix3, Point2, Point3, Rotation3, Translation, UnitVector3, Vector3}, 6 | sample_consensus::Consensus, 7 | FeatureWorldMatch, Projective, 8 | }; 9 | use lambda_twist::LambdaTwist; 10 | use rand::{rngs::SmallRng, SeedableRng}; 11 | 12 | const EPSILON_APPROX: f64 = 1e-6; 13 | 14 | fn map U, const N: usize>(f: F, array: ArrayVec) -> ArrayVec { 15 | array.into_iter().map(f).collect() 16 | } 17 | 18 | #[test] 19 | fn arrsac_manual() { 20 | let mut arrsac = Arrsac::new(0.01, SmallRng::seed_from_u64(0)); 21 | 22 | // Define some points in camera coordinates (with z > 0). 23 | let camera_depth_points: ArrayVec, 5> = map( 24 | Point3::from, 25 | [ 26 | [-0.228_125, -0.061_458_334, 1.0], 27 | [0.418_75, -0.581_25, 2.0], 28 | [1.128_125, 0.878_125, 3.0], 29 | [-0.528_125, 0.178_125, 2.5], 30 | [-0.923_424, -0.235_125, 2.8], 31 | ] 32 | .into(), 33 | ); 34 | 35 | // Define the camera pose. 36 | let rot = Rotation3::from_euler_angles(0.1, 0.2, 0.3); 37 | let trans = Translation::from(Vector3::new(0.1, 0.2, 0.3)); 38 | let pose = IsometryMatrix3::from_parts(trans, rot); 39 | 40 | // Compute world coordinates. 41 | let world_points = map(|p| pose.inverse() * p, camera_depth_points.clone()); 42 | 43 | // Compute normalized image coordinates. 44 | let normalized_image_coordinates = map(|p| (p / p.z).xy(), camera_depth_points); 45 | 46 | let samples: Vec = world_points 47 | .iter() 48 | .zip(&normalized_image_coordinates) 49 | .map(|(&world, &image)| { 50 | FeatureWorldMatch( 51 | UnitVector3::new_normalize(image.to_homogeneous()), 52 | Projective::from_homogeneous(world.to_homogeneous()), 53 | ) 54 | }) 55 | .collect(); 56 | 57 | // Estimate potential poses with P3P. 58 | // Arrsac should use the fourth point to filter and find only one model from the 4 generated. 59 | let pose = arrsac 60 | .model(&LambdaTwist::new(), samples.iter().cloned()) 61 | .unwrap(); 62 | 63 | // Compare the pose to ground truth. 64 | assert_relative_eq!(rot, pose.0.rotation, epsilon = EPSILON_APPROX); 65 | assert_relative_eq!(trans, pose.0.translation, epsilon = EPSILON_APPROX); 66 | } 67 | 68 | #[test] 69 | fn endless_loop_case() { 70 | let mut arrsac = Arrsac::new(0.01, SmallRng::seed_from_u64(0)); 71 | 72 | let samples = [ 73 | FeatureWorldMatch( 74 | UnitVector3::new_normalize( 75 | Point2::new(0.3070512144698557, 0.19317668016026052).to_homogeneous(), 76 | ), 77 | Projective::from_point(Point3::new(1.0, 1.0, 0.0)), 78 | ), 79 | FeatureWorldMatch( 80 | UnitVector3::new_normalize( 81 | Point2::new(0.3208462966353674, 0.20741702947913013).to_homogeneous(), 82 | ), 83 | Projective::from_point(Point3::new(1.0, 1.5, 0.0)), 84 | ), 85 | FeatureWorldMatch( 86 | UnitVector3::new_normalize( 87 | Point2::new(0.3070512144698557, 0.19317668016026052).to_homogeneous(), 88 | ), 89 | Projective::from_point(Point3::new(3.0, 1.0, 0.0)), 90 | ), 91 | FeatureWorldMatch( 92 | UnitVector3::new_normalize( 93 | Point2::new(0.3208462966353674, 0.20741702947913013).to_homogeneous(), 94 | ), 95 | Projective::from_point(Point3::new(1.0, 2.0, 0.0)), 96 | ), 97 | FeatureWorldMatch( 98 | UnitVector3::new_normalize( 99 | Point2::new(0.3208462966353674, 0.20741702947913013).to_homogeneous(), 100 | ), 101 | Projective::from_point(Point3::new(2.0, 2.0, 0.0)), 102 | ), 103 | FeatureWorldMatch( 104 | UnitVector3::new_normalize( 105 | Point2::new(0.3070512144698557, 0.19317668016026052).to_homogeneous(), 106 | ), 107 | Projective::from_point(Point3::new(3.0, 2.0, 0.0)), 108 | ), 109 | FeatureWorldMatch( 110 | UnitVector3::new_normalize( 111 | Point2::new(0.26619553978146293, 0.15033756455213498).to_homogeneous(), 112 | ), 113 | Projective::from_point(Point3::new(1.0, 3.0, 0.0)), 114 | ), 115 | FeatureWorldMatch( 116 | UnitVector3::new_normalize( 117 | Point2::new(0.3494806979265859, 0.18264329458710366).to_homogeneous(), 118 | ), 119 | Projective::from_point(Point3::new(2.0, 3.0, 0.0)), 120 | ), 121 | FeatureWorldMatch( 122 | UnitVector3::new_normalize( 123 | Point2::new(0.32132193890323213, 0.15408143785084824).to_homogeneous(), 124 | ), 125 | Projective::from_point(Point3::new(3.0, 3.0, 0.0)), 126 | ), 127 | ]; 128 | 129 | // Estimate potential poses with P3P. 130 | // Arrsac should use the fourth point to filter and find only one model from the 4 generated. 131 | arrsac 132 | .model(&LambdaTwist::new(), samples.iter().cloned()) 133 | .unwrap(); 134 | } 135 | -------------------------------------------------------------------------------- /cv-core/src/point.rs: -------------------------------------------------------------------------------- 1 | use derive_more::AsRef; 2 | use nalgebra::{Point3, UnitVector3, Vector4}; 3 | 4 | #[cfg(feature = "serde-serialize")] 5 | use serde::{Deserialize, Serialize}; 6 | 7 | /// This trait is implemented for homogeneous projective 3d coordinate. 8 | pub trait Projective: Clone + Copy { 9 | /// Retrieve the homogeneous vector. 10 | /// 11 | /// The homonegeous vector is guaranteed to have xyz normalized. 12 | /// The distance of the point is encoded as the reciprocal of the `w` component. 13 | /// The `w` component is guaranteed to be positive or zero. 14 | /// A `w` component of `0` implies the point is at infinity. 15 | fn homogeneous(self) -> Vector4; 16 | 17 | /// Create the projective using a homogeneous vector. 18 | /// 19 | /// This will normalize the xyz components of the provided vector and adjust `w` accordingly. 20 | fn from_homogeneous(mut point: Vector4) -> Self { 21 | if point.w.is_sign_negative() { 22 | point = -point; 23 | } 24 | Self::from_homogeneous_unchecked(point.unscale(point.xyz().norm())) 25 | } 26 | 27 | /// It is not recommended to call this directly, unless you have a good reason. 28 | /// 29 | /// The xyz components MUST be of unit length (normalized), and `w` must be adjusted accordingly. 30 | /// See [`Projective::homogeneous`] for more details. 31 | fn from_homogeneous_unchecked(point: Vector4) -> Self; 32 | 33 | /// Retrieve the euclidean 3d point by normalizing the homogeneous coordinate. 34 | /// 35 | /// This may fail, as a homogeneous coordinate can exist at near-infinity (like a star in the sky), 36 | /// whereas a 3d euclidean point cannot (it would overflow). 37 | fn point(self) -> Option> { 38 | Point3::from_homogeneous(self.homogeneous()) 39 | } 40 | 41 | /// Convert the euclidean 3d point into homogeneous coordinates. 42 | fn from_point(point: Point3) -> Self { 43 | Self::from_homogeneous(point.to_homogeneous()) 44 | } 45 | 46 | /// Retrieve the normalized bearing of the coordinate. 47 | fn bearing(self) -> UnitVector3 { 48 | UnitVector3::new_unchecked(self.homogeneous().xyz()) 49 | } 50 | } 51 | 52 | /// A 3d point in the camera's reference frame. 53 | /// 54 | /// In the camera's reference frame, the origin is the optical center, 55 | /// positive X axis is right, positive Y axis is down, and positive Z axis is forwards. 56 | /// 57 | /// The unit of distance of a `CameraPoint` is unspecified, but it should be consistent relative 58 | /// to other points in the reconstruction. 59 | #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, AsRef)] 60 | #[cfg_attr(feature = "serde-serialize", derive(Serialize, Deserialize))] 61 | pub struct CameraPoint(Vector4); 62 | 63 | impl Projective for CameraPoint { 64 | fn homogeneous(self) -> Vector4 { 65 | self.0 66 | } 67 | 68 | fn from_homogeneous_unchecked(point: Vector4) -> Self { 69 | Self(point) 70 | } 71 | } 72 | 73 | /// A point in "world" coordinates. 74 | /// This means that the real-world units of the pose are unknown, but the 75 | /// unit of distance and orientation are the same as the current reconstruction. 76 | /// 77 | /// The reason that the unit of measurement is typically unknown is because if 78 | /// the whole world is scaled by any factor `n` (excluding the camera itself), then 79 | /// the normalized image coordinates will be exactly same on every frame. Due to this, 80 | /// the scaling of the world is chosen arbitrarily. 81 | /// 82 | /// To extract the real scale of the world, a known distance between two `WorldPoint`s 83 | /// must be used to scale the whole world (and all translations between cameras). At 84 | /// that point, the world will be appropriately scaled. It is recommended not to make 85 | /// the `WorldPoint` in the reconstruction scale to the "correct" scale. This is for 86 | /// two reasons: 87 | /// 88 | /// Firstly, because it is possible for scale drift to occur due to the above situation, 89 | /// the further in the view graph you go from the reference measurement, the more the scale 90 | /// will drift from the reference. It would give a false impression that the scale is known 91 | /// globally when it is only known locally if the whole reconstruction was scaled. 92 | /// 93 | /// Secondly, as the reconstruction progresses, the reference points might get rescaled 94 | /// as optimization of the reconstruction brings everything into global consistency. 95 | /// This means that, while the reference points would be initially scaled correctly, 96 | /// any graph optimization might cause them to drift in scale as well. 97 | /// 98 | /// Please scale your points on-demand. When you need to know a real distance in the 99 | /// reconstruction, please use the closest known refenence in the view graph to scale 100 | /// it appropriately. In the future we will add APIs to utilize references 101 | /// as optimization constraints when a known reference reconstruction is present. 102 | /// 103 | /// If you must join two reconstructions, please solve for the similarity (rotation, translation and scale) 104 | /// between the two reconstructions using an optimizer. APIs will eventually be added to perform this operation 105 | /// as well. 106 | #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, AsRef)] 107 | #[cfg_attr(feature = "serde-serialize", derive(Serialize, Deserialize))] 108 | pub struct WorldPoint(pub Vector4); 109 | 110 | impl Projective for WorldPoint { 111 | fn homogeneous(self) -> Vector4 { 112 | self.0 113 | } 114 | 115 | fn from_homogeneous_unchecked(point: Vector4) -> Self { 116 | Self(point) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /tutorial-code/chapter4-feature-matching/src/main.rs: -------------------------------------------------------------------------------- 1 | use cv::{ 2 | bitarray::{BitArray, Hamming}, 3 | feature::akaze::Akaze, 4 | image::{ 5 | image::{self, DynamicImage, GenericImageView, Rgba, RgbaImage}, 6 | imageproc::drawing, 7 | }, 8 | knn::{Knn, LinearKnn}, 9 | }; 10 | use imageproc::pixelops; 11 | use itertools::Itertools; 12 | use palette::{FromColor, Hsv, RgbHue, Srgb}; 13 | 14 | fn main() { 15 | // Load the image. 16 | let src_image_a = image::open("res/0000000000.png").expect("failed to open image file"); 17 | let src_image_b = image::open("res/0000000014.png").expect("failed to open image file"); 18 | 19 | // Create an instance of `Akaze` with the default settings. 20 | let akaze = Akaze::default(); 21 | 22 | // Extract the features from the image using akaze. 23 | let (key_points_a, descriptors_a) = akaze.extract(&src_image_a); 24 | let (key_points_b, descriptors_b) = akaze.extract(&src_image_b); 25 | let matches = symmetric_matching(&descriptors_a, &descriptors_b); 26 | 27 | // Make a canvas with the `imageproc::drawing` module. 28 | // We use the blend mode so that we can draw with translucency on the image. 29 | // We convert the image to rgba8 during this process. 30 | let canvas_width = src_image_a.dimensions().0 + src_image_b.dimensions().0; 31 | let canvas_height = std::cmp::max(src_image_a.dimensions().1, src_image_b.dimensions().1); 32 | let rgba_image_a = src_image_a.to_rgba8(); 33 | let rgba_image_b = src_image_b.to_rgba8(); 34 | let mut canvas = RgbaImage::from_pixel(canvas_width, canvas_height, Rgba([0, 0, 0, 255])); 35 | 36 | // Create closure to render an image at an x offset in a canvas. 37 | let mut render_image_onto_canvas_x_offset = |image: &RgbaImage, x_offset: u32| { 38 | let (width, height) = image.dimensions(); 39 | for (x, y) in (0..width).cartesian_product(0..height) { 40 | canvas.put_pixel(x + x_offset, y, *image.get_pixel(x, y)); 41 | } 42 | }; 43 | // Render image a in the top left. 44 | render_image_onto_canvas_x_offset(&rgba_image_a, 0); 45 | // Render image b just to the right of image a (in the top right). 46 | render_image_onto_canvas_x_offset(&rgba_image_b, rgba_image_a.dimensions().0); 47 | 48 | // Draw a translucent line for every match. 49 | for (ix, &[kpa, kpb]) in matches.iter().enumerate() { 50 | // Compute a color by rotating through a color wheel on only the most saturated colors. 51 | let ix = ix as f64; 52 | let hsv = Hsv::new(RgbHue::from_radians(ix * 0.1), 1.0, 1.0); 53 | let rgb = Srgb::from_color(hsv); 54 | 55 | // Draw the line between the keypoints in the two images. 56 | let point_to_i32_tup = 57 | |point: (f32, f32), off: u32| (point.0 as i32 + off as i32, point.1 as i32); 58 | drawing::draw_antialiased_line_segment_mut( 59 | &mut canvas, 60 | point_to_i32_tup(key_points_a[kpa].point, 0), 61 | point_to_i32_tup(key_points_b[kpb].point, rgba_image_a.dimensions().0), 62 | Rgba([ 63 | (rgb.red * 255.0) as u8, 64 | (rgb.green * 255.0) as u8, 65 | (rgb.blue * 255.0) as u8, 66 | 255, 67 | ]), 68 | pixelops::interpolate, 69 | ); 70 | } 71 | 72 | // Get the resulting image. 73 | let out_image = DynamicImage::ImageRgba8(canvas); 74 | 75 | // Save the image to a temporary file. 76 | let image_file_path = tempfile::Builder::new() 77 | .suffix(".png") 78 | .tempfile() 79 | .unwrap() 80 | .into_temp_path(); 81 | out_image.save(&image_file_path).unwrap(); 82 | 83 | // Open the image with the system's default application. 84 | open::that(&image_file_path).unwrap(); 85 | // Some applications may spawn in the background and take a while to begin opening the image, 86 | // and it isn't clear if its possible to always detect whether the child process has been closed. 87 | std::thread::sleep(std::time::Duration::from_secs(5)); 88 | } 89 | 90 | /// This function performs non-symmetric matching from a to b. 91 | fn matching(a_descriptors: &[BitArray<64>], b_descriptors: &[BitArray<64>]) -> Vec> { 92 | let knn_b = LinearKnn { 93 | metric: Hamming, 94 | iter: b_descriptors.iter(), 95 | }; 96 | (0..a_descriptors.len()) 97 | .map(|a_feature| { 98 | let knn = knn_b.knn(&a_descriptors[a_feature], 2); 99 | if knn[0].distance + 24 < knn[1].distance { 100 | Some(knn[0].index) 101 | } else { 102 | None 103 | } 104 | }) 105 | .collect() 106 | } 107 | 108 | /// This function performs symmetric matching between `a` and `b`. 109 | /// 110 | /// Symmetric matching requires a feature in `b` to be the best match for a feature in `a` 111 | /// and for the same feature in `a` to be the best match for the same feature in `b`. 112 | /// The feature that a feature matches to in one direction might not be reciprocated. 113 | /// Consider a 1d line. Three features are in a line `X`, `Y`, and `Z` like `X---Y-Z`. 114 | /// `Y` is closer to `Z` than to `X`. The closest match to `X` is `Y`, but the closest 115 | /// match to `Y` is `Z`. Therefore `X` and `Y` do not match symmetrically. However, 116 | /// `Y` and `Z` do form a symmetric match, because the closest point to `Y` is `Z` 117 | /// and the closest point to `Z` is `Y`. 118 | /// 119 | /// Symmetric matching is very important for our purposes and gives stronger matches. 120 | fn symmetric_matching(a: &[BitArray<64>], b: &[BitArray<64>]) -> Vec<[usize; 2]> { 121 | // The best match for each feature in frame a to frame b's features. 122 | let forward_matches = matching(a, b); 123 | // The best match for each feature in frame b to frame a's features. 124 | let reverse_matches = matching(b, a); 125 | forward_matches 126 | .into_iter() 127 | .enumerate() 128 | .filter_map(move |(aix, bix)| { 129 | // First we only proceed if there was a sufficient bix match. 130 | // Filter out matches which are not symmetric. 131 | // Symmetric is defined as the best and sufficient match of a being b, 132 | // and likewise the best and sufficient match of b being a. 133 | bix.map(|bix| [aix, bix]) 134 | .filter(|&[aix, bix]| reverse_matches[bix] == Some(aix)) 135 | }) 136 | .collect() 137 | } 138 | -------------------------------------------------------------------------------- /tutorial/src/chapter3-akaze-feature-extraction.md: -------------------------------------------------------------------------------- 1 | # Feature extraction 2 | 3 | In this chapter, we will be writing our second Rust-CV program. Our goal will be to run the AKAZE extractor and display the result. 4 | 5 | ## What is an image feature? 6 | 7 | Features are comprised of a location in an image (called a "keypoint") and some data that helps characterize visual information about the feature (called a "descriptor"). We typically try to find features on images which can be matched to each other most easily. We want each feature to be visually discriminative and distinct so that we can find similar-looking features in other images without getting false-positives. 8 | 9 | For the purposes of Multiple-View Geometry (MVG), which encompasses Structure from Motion (SfM) and visual Simultaneous Localization and Mapping (vSLAM), we typically wish to erase certain information from the descriptor of the feature. Specifically, we want features to match so long as they correspond to the same "landmark". A landmark is a visually distinct 3d point. Since the position and orientation (known as "pose") of the camera will be different in different images, the symmetric perspective distortion, in-plane rotation, lighting, and scale of the feature might be different between different frames. For simplicity, all of these terms mean that a feature looks different when you look at it from different perspectives and at different ISO, exposure, or lighting conditions. Due to this, we typically want to erase this variable information as much as possible so that our algorithms see two features of the same landmark as the same despite these differences. 10 | 11 | If you want a more precise and accurate description of features, reading the [OpenCV documentation about features](https://docs.opencv.org/master/df/d54/tutorial_py_features_meaning.html) or Wikipedia [feature](https://en.wikipedia.org/wiki/Feature_(computer_vision)) or [descriptor](https://en.wikipedia.org/wiki/Visual_descriptor) pages is recommended. 12 | 13 | ## What is AKAZE and how does it work? 14 | 15 | In this tutorial, we use the feature extraction algoirthm [AKAZE](http://www.bmva.org/bmvc/2013/Papers/paper0013/paper0013.pdf). AKAZE attempts to be invariant to rotation, lighting, and scale. It does not attempt to be invariant towards skew, which can happen if we are looking at a surface from a large angle of incidence. AKAZE occupies a "sweet spot" among feature extraction algorithms. For one, AKAZE feature detection can pick up the interior of corners, corners, points, and blobs. AKAZE's feature matching is very nearly as robust as SIFT, the gold standard for robustness in this class of algorithm, and it often exceeds the performance of SIFT. In terms of speed, AKAZE is significantly faster than SIFT to compute, and it can be sped up substantially with the use of GPUs or [even FPGAs](http://tulipp.eu/wp-content/uploads/2019/03/2017_TUD_HEART_kalms.pdf). However, the other gold standard in this class of algorithms is [ORB](https://en.wikipedia.org/wiki/Oriented_FAST_and_rotated_BRIEF) (and the closely related [FREAK](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.446.5816&rep=rep1&type=pdf), which can perform better than ORB). This algorithm targets speed, and it is roughly 5x faster than AKAZE, although this is implementation-dependent. One downside is that ORB is significantly less robust than AKAZE. Due to these considerations, AKAZE can, given the correct algorithms and computing power, meet the needs of real-time operation while also having high quality output. 16 | 17 | AKAZE picks up features by looking at the approximation of the determinant of the hessian over the luminosity to perform a [second partial derivative test](https://en.wikipedia.org/wiki/Second_partial_derivative_test). This determinant is called the response in this context. Any local maxima in the response greater than a threshold is detected as a keypoint, and sub-pixel positioning is performed to extract the precise location of the keypoint. The threshold is typically above 0 and less than or equal to 0.01. This is done at several different scales, and at each of those scales the image is selectively blurred and occasionally shrunk. In the process of extracting the determinant of the hessian, it extracts first and second order gradients of the luminosity across X and Y in the image. By using the scale and rotation of the feature, we determine a scale and orientation at which to sample the descriptor from. The descriptor is extracted by making a series of binary comparisons in the luminosity, the first order gradients of the luminosity, and the second order gradients of the luminosity. Each binary comparison results in a single bit, and that bit is literally stored as a bit on the computer. In total, 486 comparisons are performed, thus AKAZE has a 486-bit "binary feature descriptor". For convenience, this can be padded with zeros to become 512-bit. 18 | 19 | ## Running the program 20 | 21 | Make sure you are in the Rust CV mono-repo and run: 22 | 23 | ```bash 24 | cargo run --release --bin chapter3-akaze-feature-extraction 25 | ``` 26 | 27 | If all went well you should have a window and see this: 28 | 29 | ![Akaze result](https://rust-cv.github.io/res/tutorial-images/akaze-result.png) 30 | 31 | ## The code 32 | 33 | ### Open the image 34 | 35 | ```rust 36 | let src_image = image::open("res/0000000000.png").expect("failed to open image file"); 37 | ``` 38 | 39 | We saw this in [chapter 2](./chapter2-first-program.md). This will open the image. Make sure you run this from the correct location. 40 | 41 | ### Create an AKAZE feature extractor 42 | 43 | ```rust 44 | let akaze = Akaze::default(); 45 | ``` 46 | 47 | For the purposes of this tutorial, we will just use the default AKAZE settings. You can modify the settings at this point by changing `Akaze::default()` to `Akaze::sparse()` or `Akaze::dense()`. It also has other settings you can modify as well. 48 | 49 | ### Extract the AKAZE features 50 | 51 | ```rust 52 | let (key_points, _descriptors) = akaze.extract(&src_image); 53 | ``` 54 | 55 | This line extacts the features from the image. In this case, we will not be using the descriptor data, so those are discarded with the `_`. 56 | 57 | ### Draw crosses and show image 58 | 59 | ```rust 60 | for KeyPoint { point: (x, y), .. } in key_points { 61 | drawing::draw_cross_mut( 62 | &mut image_canvas, 63 | Rgba([0, 255, 255, 128]), 64 | x as i32, 65 | y as i32, 66 | ); 67 | } 68 | ``` 69 | 70 | Almost all of the rest of the code is the same as what we saw in [chapter 2](./chapter2-first-program.md). However, the above snippet is slightly different. Rather than randomly generating points, we are now using the X and Y components of the keypoints AKAZE extracted. The output image actually shows the keypoints of the features AKAZE found. 71 | 72 | ## End 73 | 74 | This is the end of this chapter. 75 | 76 | -------------------------------------------------------------------------------- /vslam-sandbox/src/main.rs: -------------------------------------------------------------------------------- 1 | use arrsac::Arrsac; 2 | use cv_core::nalgebra::{Point2, Vector2}; 3 | use cv_geom::triangulation::LinearEigenTriangulator; 4 | use cv_pinhole::{CameraIntrinsics, CameraIntrinsicsK1Distortion}; 5 | use cv_sfm::{VSlam, VSlamSettings}; 6 | use eight_point::EightPoint; 7 | use lambda_twist::LambdaTwist; 8 | 9 | use log::*; 10 | use rand::SeedableRng; 11 | use rand_xoshiro::Xoshiro256PlusPlus; 12 | use slotmap::Key; 13 | use std::{collections::HashSet, path::PathBuf}; 14 | use structopt::StructOpt; 15 | 16 | #[derive(StructOpt, Clone)] 17 | #[structopt(name = "vslam-sandbox", about = "A tool for testing vslam algorithms")] 18 | struct Opt { 19 | /// The file where reconstruction data is accumulated. 20 | /// 21 | /// If this file doesn't exist, the file will be created when the program finishes. 22 | #[structopt(short, long, default_value = "vslam.cvr")] 23 | data: PathBuf, 24 | /// The file where settings are specified. 25 | /// 26 | /// This is in the format of `cv_reconstruction::VSlamSettings`. 27 | #[structopt(short, long, default_value = "vslam-settings.json")] 28 | settings: PathBuf, 29 | /// The maximum cosine distance an observation can have to be exported. 30 | #[structopt(long, default_value = "0.000001")] 31 | export_maximum_cosine_distance: f64, 32 | /// Export required observations 33 | #[structopt(long, default_value = "3")] 34 | export_robust_minimum_observations: usize, 35 | /// The x focal length for "The Zurich Urban Micro Aerial Vehicle Dataset" 36 | #[structopt(long, default_value = "893.39010814")] 37 | x_focal: f64, 38 | /// The y focal length for "The Zurich Urban Micro Aerial Vehicle Dataset" 39 | #[structopt(long, default_value = "898.32648616")] 40 | y_focal: f64, 41 | /// The x optical center coordinate for "The Zurich Urban Micro Aerial Vehicle Dataset" 42 | #[structopt(long, default_value = "951.1310043")] 43 | x_center: f64, 44 | /// The y optical center coordinate for "The Zurich Urban Micro Aerial Vehicle Dataset" 45 | #[structopt(long, default_value = "555.13350077")] 46 | y_center: f64, 47 | /// The skew for "The Zurich Urban Micro Aerial Vehicle Dataset" 48 | #[structopt(long, default_value = "0.0")] 49 | skew: f64, 50 | /// The K1 radial distortion for "The Zurich Urban Micro Aerial Vehicle Dataset" 51 | #[structopt(long, default_value = "-0.28052513")] 52 | radial_distortion: f64, 53 | /// If true, will use actual PLY faces for cameras rather than just points. 54 | #[structopt(long)] 55 | no_camera_faces: bool, 56 | /// Output directory for reconstruction PLY files 57 | #[structopt(short, long)] 58 | output: Option, 59 | /// List of image files 60 | /// 61 | /// Default vales are for "The Zurich Urban Micro Aerial Vehicle Dataset" 62 | #[structopt(parse(from_os_str))] 63 | images: Vec, 64 | } 65 | 66 | fn main() { 67 | pretty_env_logger::init_timed(); 68 | let opt = Opt::from_args(); 69 | 70 | // Fill intrinsics from args. 71 | let intrinsics = CameraIntrinsicsK1Distortion::new( 72 | CameraIntrinsics { 73 | focals: Vector2::new(opt.x_focal, opt.y_focal), 74 | principal_point: Point2::new(opt.x_center, opt.y_center), 75 | skew: opt.skew, 76 | }, 77 | opt.radial_distortion, 78 | ); 79 | 80 | info!("trying to load existing reconstruction data"); 81 | let data = std::fs::File::open(&opt.data) 82 | .ok() 83 | .map(|file| bincode::deserialize_from(file).expect("failed to deserialize reconstruction")); 84 | if data.is_some() { 85 | info!("loaded existing reconstruction"); 86 | } else { 87 | info!("used empty reconstruction"); 88 | } 89 | let data = data.unwrap_or_default(); 90 | 91 | let settings = std::fs::File::open(&opt.settings) 92 | .ok() 93 | .and_then(|file| serde_json::from_reader(file).ok()); 94 | if settings.is_some() { 95 | info!("loaded existing settings"); 96 | } else { 97 | info!("used default settings"); 98 | } 99 | let settings: VSlamSettings = settings.unwrap_or_default(); 100 | 101 | // Create a channel that will produce features in another parallel thread. 102 | let mut vslam = VSlam::new( 103 | data, 104 | settings, 105 | Arrsac::new( 106 | settings.single_view_consensus_threshold, 107 | Xoshiro256PlusPlus::seed_from_u64(0), 108 | ) 109 | .initialization_hypotheses(16384) 110 | .max_candidate_hypotheses(1024) 111 | .estimations_per_block(256), 112 | Arrsac::new( 113 | settings.two_view_consensus_threshold, 114 | Xoshiro256PlusPlus::seed_from_u64(0), 115 | ) 116 | .initialization_hypotheses(8192) 117 | .max_candidate_hypotheses(1024), 118 | LambdaTwist::new(), 119 | EightPoint::new(), 120 | LinearEigenTriangulator::new(), 121 | Xoshiro256PlusPlus::seed_from_u64(0), 122 | ); 123 | 124 | // Add the feed. 125 | let feed = vslam.add_feed(intrinsics); 126 | 127 | let mut normalized = HashSet::new(); 128 | 129 | // Add the frames. 130 | for frame_path in &opt.images { 131 | info!("loading image {}", frame_path.display()); 132 | let image = image::open(frame_path).expect("failed to load image"); 133 | let frame = vslam.add_frame(feed, &image); 134 | if let Some((reconstruction, _)) = vslam.data.frame(frame).view { 135 | if normalized.insert(reconstruction) { 136 | info!("new reconstruction; normalizing reconstruction"); 137 | vslam.normalize_reconstruction(reconstruction); 138 | } 139 | info!("exporting reconstruction"); 140 | if let Some(path) = &opt.output { 141 | if !path.is_dir() { 142 | warn!( 143 | "output path is not a directory; it must be a directory; skipping export" 144 | ); 145 | } else { 146 | // Keep track of the old settings 147 | let old_settings = vslam.settings; 148 | // Set the settings based on the command line arguments for export purposes. 149 | vslam.settings.maximum_cosine_distance = opt.export_maximum_cosine_distance; 150 | vslam.settings.robust_minimum_observations = 151 | opt.export_robust_minimum_observations; 152 | let path = path.join(format!( 153 | "{}_{}.ply", 154 | frame_path.file_name().unwrap().to_str().unwrap(), 155 | reconstruction.data().as_ffi() 156 | )); 157 | vslam.export_reconstruction(reconstruction, &path, !opt.no_camera_faces); 158 | info!("exported {}", path.display()); 159 | // Restore the pre-export settings. 160 | vslam.settings = old_settings; 161 | } 162 | } 163 | } 164 | } 165 | 166 | if !opt.images.is_empty() { 167 | info!("saving the reconstruction data"); 168 | if let Ok(file) = std::fs::File::create(opt.data) { 169 | if let Err(e) = bincode::serialize_into(file, &vslam.data) { 170 | error!("unable to save reconstruction data: {}", e); 171 | } 172 | } 173 | } else { 174 | info!("reconstruction not modified, so not saving reconstruction data"); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /akaze/src/descriptors.rs: -------------------------------------------------------------------------------- 1 | use crate::{Akaze, Error, EvolutionStep, KeyPoint}; 2 | use bitarray::BitArray; 3 | 4 | #[cfg(feature = "rayon")] 5 | use rayon::prelude::*; 6 | 7 | impl Akaze { 8 | /// Extract descriptors from keypoints/an evolution 9 | /// 10 | /// # Arguments 11 | /// * `evolutions` - the nonlinear scale space 12 | /// * `keypoints` - the keypoints detected. 13 | /// * `options` - The options of the nonlinear scale space. 14 | /// # Return value 15 | /// A vector of descriptors. 16 | pub fn extract_descriptors( 17 | &self, 18 | evolutions: &[EvolutionStep], 19 | keypoints: &[KeyPoint], 20 | ) -> (Vec, Vec>) { 21 | #[cfg(not(feature = "rayon"))] 22 | { 23 | keypoints 24 | .iter() 25 | .filter_map(|&keypoint| { 26 | Some(( 27 | keypoint, 28 | self.get_mldb_descriptor(&keypoint, evolutions).ok()?, 29 | )) 30 | }) 31 | .unzip() 32 | } 33 | #[cfg(feature = "rayon")] 34 | { 35 | keypoints 36 | .par_iter() 37 | .filter_map(|&keypoint| { 38 | Some(( 39 | keypoint, 40 | self.get_mldb_descriptor(&keypoint, evolutions).ok()?, 41 | )) 42 | }) 43 | .unzip() 44 | } 45 | } 46 | 47 | /// Computes the rotation invariant M-LDB binary descriptor (maximum descriptor length) 48 | /// 49 | /// # Arguments 50 | /// `* kpt` - Input keypoint 51 | /// * `evolutions` - Input evolutions 52 | /// * `options` - Input options 53 | /// # Return value 54 | /// Binary-based descriptor 55 | fn get_mldb_descriptor( 56 | &self, 57 | keypoint: &KeyPoint, 58 | evolutions: &[EvolutionStep], 59 | ) -> Result, Error> { 60 | let mut output = BitArray::zeros(); 61 | let max_channels = 3usize; 62 | debug_assert!(self.descriptor_channels <= max_channels); 63 | let mut values: Vec = vec![0f32; 16 * max_channels]; 64 | let size_mult = [1.0f32, 2.0f32 / 3.0f32, 1.0f32 / 2.0f32]; 65 | 66 | let ratio = (1u32 << keypoint.octave) as f32; 67 | let scale = f32::round(0.5f32 * keypoint.size / ratio); 68 | let xf = keypoint.point.0 / ratio; 69 | let yf = keypoint.point.1 / ratio; 70 | let co = f32::cos(keypoint.angle); 71 | let si = f32::sin(keypoint.angle); 72 | let pattern_size = self.descriptor_pattern_size as f32; 73 | 74 | let mut dpos = 0usize; 75 | for (lvl, multiplier) in size_mult.iter().enumerate() { 76 | let val_count = (lvl + 2usize) * (lvl + 2usize); 77 | let sample_size = f32::ceil(pattern_size * multiplier) as usize; 78 | self.mldb_fill_values( 79 | &mut values, 80 | sample_size, 81 | keypoint.class_id, 82 | xf, 83 | yf, 84 | co, 85 | si, 86 | scale, 87 | evolutions, 88 | )?; 89 | mldb_binary_comparisons( 90 | &values, 91 | output.bytes_mut(), 92 | val_count, 93 | &mut dpos, 94 | self.descriptor_channels, 95 | ); 96 | } 97 | Ok(output) 98 | } 99 | 100 | /// Fill the comparison values for the MLDB rotation invariant descriptor 101 | #[allow(clippy::too_many_arguments)] 102 | fn mldb_fill_values( 103 | &self, 104 | values: &mut [f32], 105 | sample_step: usize, 106 | level: usize, 107 | xf: f32, 108 | yf: f32, 109 | co: f32, 110 | si: f32, 111 | scale: f32, 112 | evolutions: &[EvolutionStep], 113 | ) -> Result<(), Error> { 114 | let pattern_size = self.descriptor_pattern_size; 115 | let nr_channels = self.descriptor_channels; 116 | let mut valuepos = 0; 117 | for i in (-(pattern_size as i32)..(pattern_size as i32)).step_by(sample_step) { 118 | for j in (-(pattern_size as i32)..(pattern_size as i32)).step_by(sample_step) { 119 | let mut di = 0f32; 120 | let mut dx = 0f32; 121 | let mut dy = 0f32; 122 | let mut nsamples = 0usize; 123 | for k in i..(i + (sample_step as i32)) { 124 | for l in j..(j + (sample_step as i32)) { 125 | let l = l as f32; 126 | let k = k as f32; 127 | let sample_y = yf + (l * co * scale + k * si * scale); 128 | let sample_x = xf + (-l * si * scale + k * co * scale); 129 | let y1 = f32::round(sample_y) as isize; 130 | let x1 = f32::round(sample_x) as isize; 131 | if !(0..evolutions[level].Lt.width() as isize).contains(&x1) 132 | || !(0..evolutions[level].Lt.height() as isize).contains(&y1) 133 | { 134 | return Err(Error::SampleOutOfBounds { 135 | x: x1, 136 | y: y1, 137 | width: evolutions[level].Lt.width(), 138 | height: evolutions[level].Lt.height(), 139 | }); 140 | } 141 | let y1 = y1 as usize; 142 | let x1 = x1 as usize; 143 | let ri = evolutions[level].Lt.get(x1, y1); 144 | di += ri; 145 | if nr_channels > 1 { 146 | let rx = evolutions[level].Lx.get(x1, y1); 147 | let ry = evolutions[level].Ly.get(x1, y1); 148 | if nr_channels == 2 { 149 | dx += f32::sqrt(rx * rx + ry * ry); 150 | } else { 151 | let rry = rx * co + ry * si; 152 | let rrx = -rx * si + ry * co; 153 | dx += rrx; 154 | dy += rry; 155 | } 156 | } 157 | nsamples += 1; 158 | } 159 | } 160 | 161 | di /= nsamples as f32; 162 | dx /= nsamples as f32; 163 | dy /= nsamples as f32; 164 | 165 | values[valuepos] = di; 166 | 167 | if nr_channels > 1 { 168 | values[valuepos + 1] = dx; 169 | } 170 | if nr_channels > 2 { 171 | values[valuepos + 2] = dy; 172 | } 173 | valuepos += nr_channels; 174 | } 175 | } 176 | Ok(()) 177 | } 178 | } 179 | 180 | /// Do the binary comparisons to obtain the descriptor 181 | fn mldb_binary_comparisons( 182 | values: &[f32], 183 | descriptor: &mut [u8], 184 | count: usize, 185 | dpos: &mut usize, 186 | nr_channels: usize, 187 | ) { 188 | for pos in 0..nr_channels { 189 | for i in 0..count { 190 | let ival = values[nr_channels * i + pos]; 191 | for j in (i + 1)..count { 192 | let res = if ival > values[nr_channels * j + pos] { 193 | 1u8 194 | } else { 195 | 0u8 196 | }; 197 | descriptor[*dpos >> 3usize] |= res << (*dpos & 7); 198 | *dpos += 1usize; 199 | } 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /tutorial-code/chapter5-geometric-verification/src/main.rs: -------------------------------------------------------------------------------- 1 | use cv::{ 2 | bitarray::{BitArray, Hamming}, 3 | camera::pinhole::{CameraIntrinsics, CameraIntrinsicsK1Distortion}, 4 | consensus::Arrsac, 5 | estimate::EightPoint, 6 | feature::akaze::Akaze, 7 | image::{ 8 | image::{self, DynamicImage, GenericImageView, Rgba, RgbaImage}, 9 | imageproc::drawing, 10 | }, 11 | knn::{Knn, LinearKnn}, 12 | nalgebra::{Point2, Vector2}, 13 | sample_consensus::Consensus, 14 | CameraModel, FeatureMatch, KeyPoint, Pose, 15 | }; 16 | use imageproc::pixelops; 17 | use itertools::Itertools; 18 | use palette::{FromColor, Hsv, RgbHue, Srgb}; 19 | use rand::SeedableRng; 20 | use rand_xoshiro::Xoshiro256PlusPlus; 21 | 22 | fn main() { 23 | // Load the image. 24 | let src_image_a = image::open("res/0000000000.png").expect("failed to open image file"); 25 | let src_image_b = image::open("res/0000000014.png").expect("failed to open image file"); 26 | 27 | // Create an instance of `Akaze` with the default settings. 28 | let akaze = Akaze::default(); 29 | 30 | // Extract the features from the image using akaze. 31 | let (key_points_a, descriptors_a) = akaze.extract(&src_image_a); 32 | let (key_points_b, descriptors_b) = akaze.extract(&src_image_b); 33 | let matches = symmetric_matching(&descriptors_a, &descriptors_b); 34 | 35 | // The camera calibration data for 2011_09_26_drive_0035 camera 0 of the Kitti dataset. 36 | let camera = CameraIntrinsicsK1Distortion::new( 37 | CameraIntrinsics::identity() 38 | .focals(Vector2::new(9.842439e+02, 9.808141e+02)) 39 | .principal_point(Point2::new(6.900000e+02, 2.331966e+02)), 40 | -3.728755e-01, 41 | ); 42 | 43 | // Create an instance of ARRSAC, the consensus algorithm that will find 44 | // a model that best fits the data. We need to pass in an RNG with good statistical properties 45 | // for the random sampling process, and xoshiro256++ is an excellent PRNG for this purpose. 46 | // It is prefered for this example to use a PRNG so we get the same result every time. 47 | // Note that the inlier threshold is set to 1e-7. This is specific to the dataset. 48 | let mut consensus = Arrsac::new(1e-7, Xoshiro256PlusPlus::seed_from_u64(0)); 49 | 50 | // Create the estimator. In this case it is the well-known Eight-Point algorithm. 51 | let estimator = EightPoint::new(); 52 | 53 | // Take all of the original matches and use the camera model to compute the bearing 54 | // of each keypoint. The bearing is the direction that the light came from in the camera's 55 | // reference frame. 56 | let matches = matches 57 | .iter() 58 | .map(|&[a, b]| { 59 | FeatureMatch( 60 | camera.calibrate(key_points_a[a]), 61 | camera.calibrate(key_points_b[b]), 62 | ) 63 | }) 64 | .collect_vec(); 65 | 66 | // Run the consensus process. This will use the estimator to estimate the pose of the camera 67 | // from random data repeatedly. It does this in an intelligent way to maximize the number of 68 | // inliers to the model. For convenience, we use .expect() since we expect to get a pose back, 69 | // but this should not be used in real code. 70 | let (pose, inliers) = consensus 71 | .model_inliers(&estimator, matches.iter().copied()) 72 | .expect("we expect to get a pose"); 73 | 74 | // Print out the direction the camera moved. 75 | // Note that the translation of the pose is in the final camera's reference frame 76 | // and describes the direction that the point cloud (the world) moves to become that reference 77 | // frame. Therefore, the negation of the translation of the isometry is the actual 78 | // translation of the camera. 79 | let translation = -pose.isometry().translation.vector; 80 | println!("camera moved forward: {}", translation.z); 81 | println!("camera moved right: {}", translation.x); 82 | println!("camera moved down: {}", translation.y); 83 | 84 | // Only keep the inlier matches. 85 | let matches = inliers.iter().map(|&inlier| matches[inlier]).collect_vec(); 86 | 87 | // Make a canvas with the `imageproc::drawing` module. 88 | // We use the blend mode so that we can draw with translucency on the image. 89 | // We convert the image to rgba8 during this process. 90 | let canvas_width = src_image_a.dimensions().0 + src_image_b.dimensions().0; 91 | let canvas_height = std::cmp::max(src_image_a.dimensions().1, src_image_b.dimensions().1); 92 | let rgba_image_a = src_image_a.to_rgba8(); 93 | let rgba_image_b = src_image_b.to_rgba8(); 94 | let mut canvas = RgbaImage::from_pixel(canvas_width, canvas_height, Rgba([0, 0, 0, 255])); 95 | 96 | // Create closure to render an image at an x offset in a canvas. 97 | let mut render_image_onto_canvas_x_offset = |image: &RgbaImage, x_offset: u32| { 98 | let (width, height) = image.dimensions(); 99 | for (x, y) in (0..width).cartesian_product(0..height) { 100 | canvas.put_pixel(x + x_offset, y, *image.get_pixel(x, y)); 101 | } 102 | }; 103 | // Render image a in the top left. 104 | render_image_onto_canvas_x_offset(&rgba_image_a, 0); 105 | // Render image b just to the right of image a (in the top right). 106 | render_image_onto_canvas_x_offset(&rgba_image_b, rgba_image_a.dimensions().0); 107 | 108 | // Draw a translucent line for every match. 109 | for (ix, &FeatureMatch(a_bearing, b_bearing)) in matches.iter().enumerate() { 110 | // Compute a color by rotating through a color wheel on only the most saturated colors. 111 | let ix = ix as f64; 112 | let hsv = Hsv::new(RgbHue::from_radians(ix * 0.1), 1.0, 1.0); 113 | let rgb = Srgb::from_color(hsv); 114 | 115 | // Draw the line between the keypoints in the two images. 116 | let point_to_i32_tup = 117 | |point: KeyPoint, off: u32| (point.x as i32 + off as i32, point.y as i32); 118 | drawing::draw_antialiased_line_segment_mut( 119 | &mut canvas, 120 | point_to_i32_tup(camera.uncalibrate(a_bearing).unwrap(), 0), 121 | point_to_i32_tup( 122 | camera.uncalibrate(b_bearing).unwrap(), 123 | rgba_image_a.dimensions().0, 124 | ), 125 | Rgba([ 126 | (rgb.red * 255.0) as u8, 127 | (rgb.green * 255.0) as u8, 128 | (rgb.blue * 255.0) as u8, 129 | 255, 130 | ]), 131 | pixelops::interpolate, 132 | ); 133 | } 134 | 135 | // Get the resulting image. 136 | let out_image = DynamicImage::ImageRgba8(canvas); 137 | 138 | // Save the image to a temporary file. 139 | let image_file_path = tempfile::Builder::new() 140 | .suffix(".png") 141 | .tempfile() 142 | .unwrap() 143 | .into_temp_path(); 144 | out_image.save(&image_file_path).unwrap(); 145 | 146 | // Open the image with the system's default application. 147 | open::that(&image_file_path).unwrap(); 148 | // Some applications may spawn in the background and take a while to begin opening the image, 149 | // and it isn't clear if its possible to always detect whether the child process has been closed. 150 | std::thread::sleep(std::time::Duration::from_secs(5)); 151 | } 152 | 153 | /// This function performs non-symmetric matching from a to b. 154 | fn matching(a_descriptors: &[BitArray<64>], b_descriptors: &[BitArray<64>]) -> Vec> { 155 | let knn_b = LinearKnn { 156 | metric: Hamming, 157 | iter: b_descriptors.iter(), 158 | }; 159 | (0..a_descriptors.len()) 160 | .map(|a_feature| { 161 | let knn = knn_b.knn(&a_descriptors[a_feature], 2); 162 | if knn[0].distance + 24 < knn[1].distance { 163 | Some(knn[0].index) 164 | } else { 165 | None 166 | } 167 | }) 168 | .collect() 169 | } 170 | 171 | /// This function performs symmetric matching between `a` and `b`. 172 | /// 173 | /// Symmetric matching requires a feature in `b` to be the best match for a feature in `a` 174 | /// and for the same feature in `a` to be the best match for the same feature in `b`. 175 | /// The feature that a feature matches to in one direction might not be reciprocated. 176 | /// Consider a 1d line. Three features are in a line `X`, `Y`, and `Z` like `X---Y-Z`. 177 | /// `Y` is closer to `Z` than to `X`. The closest match to `X` is `Y`, but the closest 178 | /// match to `Y` is `Z`. Therefore `X` and `Y` do not match symmetrically. However, 179 | /// `Y` and `Z` do form a symmetric match, because the closest point to `Y` is `Z` 180 | /// and the closest point to `Z` is `Y`. 181 | /// 182 | /// Symmetric matching is very important for our purposes and gives stronger matches. 183 | fn symmetric_matching(a: &[BitArray<64>], b: &[BitArray<64>]) -> Vec<[usize; 2]> { 184 | // The best match for each feature in frame a to frame b's features. 185 | let forward_matches = matching(a, b); 186 | // The best match for each feature in frame b to frame a's features. 187 | let reverse_matches = matching(b, a); 188 | forward_matches 189 | .into_iter() 190 | .enumerate() 191 | .filter_map(move |(aix, bix)| { 192 | // First we only proceed if there was a sufficient bix match. 193 | // Filter out matches which are not symmetric. 194 | // Symmetric is defined as the best and sufficient match of a being b, 195 | // and likewise the best and sufficient match of b being a. 196 | bix.map(|bix| [aix, bix]) 197 | .filter(|&[aix, bix]| reverse_matches[bix] == Some(aix)) 198 | }) 199 | .collect() 200 | } 201 | -------------------------------------------------------------------------------- /cv-core/src/so3.rs: -------------------------------------------------------------------------------- 1 | use core::{ 2 | iter::Sum, 3 | ops::{Add, AddAssign}, 4 | }; 5 | use derive_more::{AsMut, AsRef, Deref, DerefMut, From, Into}; 6 | use nalgebra::{Const, IsometryMatrix3, Matrix3, Matrix4, Rotation3, Unit, Vector3, Vector6}; 7 | use num_traits::Float; 8 | #[cfg(feature = "serde-serialize")] 9 | use serde::{Deserialize, Serialize}; 10 | 11 | /// Contains a small gradient translation and rotation that will be appended to 12 | /// the reference frame of some pose. 13 | /// 14 | /// This is a member of the lie algebra se(3). 15 | #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] 16 | pub struct Se3TangentSpace { 17 | pub translation: Vector3, 18 | pub rotation: Vector3, 19 | } 20 | 21 | impl Se3TangentSpace { 22 | #[inline(always)] 23 | pub fn new(mut translation: Vector3, mut rotation: Vector3) -> Self { 24 | if translation.iter().any(|n| n.is_nan()) { 25 | translation = Vector3::zeros(); 26 | } 27 | if rotation.iter().any(|n| n.is_nan()) { 28 | rotation = Vector3::zeros(); 29 | } 30 | Self { 31 | translation, 32 | rotation, 33 | } 34 | } 35 | 36 | #[inline(always)] 37 | pub fn identity() -> Self { 38 | Self { 39 | translation: Vector3::zeros(), 40 | rotation: Vector3::zeros(), 41 | } 42 | } 43 | 44 | /// Inverts the transformation, which is very cheap. 45 | #[must_use] 46 | #[inline(always)] 47 | pub fn inverse(self) -> Self { 48 | Self { 49 | translation: -self.translation, 50 | rotation: -self.rotation, 51 | } 52 | } 53 | 54 | /// Gets the isometry that represents this tangent space transformation. 55 | #[must_use] 56 | #[inline(always)] 57 | pub fn isometry(self) -> IsometryMatrix3 { 58 | let rotation = Rotation3::from_scaled_axis(self.rotation); 59 | IsometryMatrix3::from_parts((rotation * self.translation).into(), rotation) 60 | } 61 | 62 | /// For tangent spaces where the translation and rotation are both rotational, this retrieves the 63 | /// translation and rotation rotation matrix. The rotation matrix for the translation rotates the translation, 64 | /// while the rotation matrix for the rotation is left-multiplied by the rotation. 65 | /// 66 | /// Returns `(translation_rotation, rotation)`. 67 | #[must_use] 68 | #[inline(always)] 69 | pub fn rotations(self) -> (Rotation3, Rotation3) { 70 | let translation_rotation = Rotation3::from_scaled_axis(self.translation); 71 | let rotation = Rotation3::from_scaled_axis(self.rotation); 72 | (translation_rotation, rotation) 73 | } 74 | 75 | /// Scales both the rotation and the translation. 76 | #[must_use] 77 | #[inline(always)] 78 | pub fn scale(mut self, scale: f64) -> Self { 79 | self.translation *= scale; 80 | self.rotation *= scale; 81 | self 82 | } 83 | 84 | /// Scales the translation. 85 | #[must_use] 86 | #[inline(always)] 87 | pub fn scale_translation(mut self, scale: f64) -> Self { 88 | self.translation *= scale; 89 | self 90 | } 91 | 92 | /// Scales the rotation. 93 | #[must_use] 94 | #[inline(always)] 95 | pub fn scale_rotation(mut self, scale: f64) -> Self { 96 | self.rotation *= scale; 97 | self 98 | } 99 | 100 | #[inline(always)] 101 | pub fn to_vec(&self) -> Vector6 { 102 | Vector6::new( 103 | self.translation.x, 104 | self.translation.y, 105 | self.translation.z, 106 | self.rotation.x, 107 | self.rotation.y, 108 | self.rotation.z, 109 | ) 110 | } 111 | 112 | #[inline(always)] 113 | pub fn from_vec(v: Vector6) -> Self { 114 | Self { 115 | translation: v.rows_generic(0, Const::<3>).into_owned(), 116 | rotation: v.rows_generic(3, Const::<3>).into_owned(), 117 | } 118 | } 119 | 120 | /// Assumes an L2 tangent space is provided as input and returns the L1 tangent space. 121 | #[inline(always)] 122 | #[must_use] 123 | pub fn l1(&self) -> Self { 124 | Self::new(self.translation.normalize(), self.rotation.normalize()) 125 | } 126 | } 127 | 128 | impl Add for Se3TangentSpace { 129 | type Output = Self; 130 | 131 | fn add(self, rhs: Self) -> Self { 132 | Self { 133 | translation: self.translation + rhs.translation, 134 | rotation: self.rotation + rhs.rotation, 135 | } 136 | } 137 | } 138 | 139 | impl AddAssign for Se3TangentSpace { 140 | fn add_assign(&mut self, rhs: Self) { 141 | self.translation += rhs.translation; 142 | self.rotation += rhs.rotation; 143 | } 144 | } 145 | 146 | impl Sum for Se3TangentSpace { 147 | fn sum>(iter: I) -> Self { 148 | iter.fold(Se3TangentSpace::identity(), |a, b| a + b) 149 | } 150 | } 151 | 152 | /// Contains a member of the lie algebra so(3), a representation of the tangent space 153 | /// of 3d rotation. This is also known as the lie algebra of the 3d rotation group SO(3). 154 | /// 155 | /// This is only intended to be used in optimization problems where it is desirable to 156 | /// have unconstranied variables representing the degrees of freedom of the rotation. 157 | /// In all other cases, a rotation matrix should be used to store rotations, since the 158 | /// conversion to and from a rotation matrix is non-trivial. 159 | #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, AsMut, AsRef, Deref, DerefMut, From, Into)] 160 | #[cfg_attr(feature = "serde-serialize", derive(Serialize, Deserialize))] 161 | pub struct Skew3(pub Vector3); 162 | 163 | impl Skew3 { 164 | /// Converts the Skew3 to a Rotation3 matrix. 165 | pub fn rotation(self) -> Rotation3 { 166 | self.into() 167 | } 168 | 169 | /// Converts the Skew3 into a Rotation3 matrix quickly, but only works when the rotation 170 | /// is very small. 171 | pub fn rotation_small(self) -> Rotation3 { 172 | Rotation3::from_matrix(&(Matrix3::identity() + self.hat())) 173 | } 174 | 175 | /// This converts a matrix in skew-symmetric form into a Skew3. 176 | /// 177 | /// Warning: Does no check to ensure matrix is actually skew-symmetric. 178 | pub fn vee(mat: Matrix3) -> Self { 179 | Self(Vector3::new(mat.m32, mat.m13, mat.m21)) 180 | } 181 | 182 | /// This converts the Skew3 into its skew-symmetric matrix form. 183 | #[rustfmt::skip] 184 | pub fn hat(self) -> Matrix3 { 185 | self.0.cross_matrix() 186 | } 187 | 188 | /// This converts the Skew3 into its squared skew-symmetric matrix form efficiently. 189 | #[rustfmt::skip] 190 | pub fn hat2(self) -> Matrix3 { 191 | let w = self.0; 192 | let w11 = w.x * w.x; 193 | let w12 = w.x * w.y; 194 | let w13 = w.x * w.z; 195 | let w22 = w.y * w.y; 196 | let w23 = w.y * w.z; 197 | let w33 = w.z * w.z; 198 | Matrix3::new( 199 | -w22 - w33, w12, w13, 200 | w12, -w11 - w33, w23, 201 | w13, w23, -w11 - w22, 202 | ) 203 | } 204 | 205 | /// Computes the lie bracket [self, rhs]. 206 | #[must_use] 207 | pub fn bracket(self, rhs: Self) -> Self { 208 | Self::vee(self.hat() * rhs.hat() - rhs.hat() * self.hat()) 209 | } 210 | 211 | /// The jacobian of the output of a rotation in respect to the 212 | /// input of a rotation. 213 | /// 214 | /// `y = R * x` 215 | /// 216 | /// `dy/dx = R` 217 | /// 218 | /// The formula is pretty simple and is just the rotation matrix created 219 | /// from the exponential map of this so(3) element into SO(3). The result is converted 220 | /// to homogeneous form (by adding a new dimension with a `1` in the diagonal) so 221 | /// that it is compatible with homogeneous coordinates. 222 | /// 223 | /// If you have the rotation matrix already, please use the rotation matrix itself 224 | /// rather than calling this method. Calling this method will waste time converting 225 | /// the [`Skew3`] back into a [`Rotation3`], which is non-trivial. 226 | pub fn jacobian_input(self) -> Matrix4 { 227 | let rotation: Rotation3 = self.into(); 228 | let matrix: Matrix3 = rotation.into(); 229 | matrix.to_homogeneous() 230 | } 231 | 232 | /// The jacobian of the output of a rotation in respect to the 233 | /// rotation itself. 234 | /// 235 | /// `y = R * x` 236 | /// 237 | /// `dy/dR = -hat(y)` 238 | /// 239 | /// The derivative is purely based on the current output vector, and thus doesn't take `self`. 240 | /// 241 | /// Note that when working with homogeneous projective coordinates, only the first three components 242 | /// (the bearing) are relevant, hence the resulting matrix is a [`Matrix3`]. 243 | pub fn jacobian_self(y: Vector3) -> Matrix3 { 244 | y.cross_matrix() 245 | } 246 | } 247 | 248 | /// This is the exponential map. 249 | impl From for Rotation3 { 250 | fn from(w: Skew3) -> Self { 251 | // This check is done to avoid the degenerate case where the angle is near zero. 252 | let theta2 = w.0.norm_squared(); 253 | if theta2 <= f64::epsilon() { 254 | w.rotation_small() 255 | } else { 256 | let theta = theta2.sqrt(); 257 | let axis = Unit::new_unchecked(w.0 / theta); 258 | Self::from_axis_angle(&axis, theta) 259 | } 260 | } 261 | } 262 | 263 | /// This is the log map. 264 | impl From> for Skew3 { 265 | fn from(r: Rotation3) -> Self { 266 | let skew3 = r.scaled_axis(); 267 | // TODO: File issue on `nalgebra`, as this shouldn't happen and is bug. 268 | let skew3 = if skew3.iter().any(|n| n.is_nan()) { 269 | Vector3::zeros() 270 | } else { 271 | skew3 272 | }; 273 | Self(skew3) 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /cv-optimize/src/three_view_optimizer.rs: -------------------------------------------------------------------------------- 1 | use cv_core::{ 2 | nalgebra::{IsometryMatrix3, UnitVector3}, 3 | CameraToCamera, Pose, Se3TangentSpace, 4 | }; 5 | use cv_geom::epipolar; 6 | 7 | fn landmark_gradients( 8 | poses: [IsometryMatrix3; 2], 9 | observations: [UnitVector3; 3], 10 | ) -> [Se3TangentSpace; 2] { 11 | let ftoc = poses[0]; 12 | let stoc = poses[1]; 13 | 14 | epipolar::three_view_gradients( 15 | observations[0], 16 | ftoc * observations[1], 17 | ftoc.translation.vector, 18 | stoc * observations[2], 19 | stoc.translation.vector, 20 | ) 21 | } 22 | 23 | pub fn three_view_simple_optimize_l1( 24 | poses: [CameraToCamera; 2], 25 | epsilon: f64, 26 | optimization_rate: f64, 27 | iterations: usize, 28 | landmarks: &[[UnitVector3; 3]], 29 | ) -> [CameraToCamera; 2] { 30 | if landmarks.is_empty() { 31 | return poses; 32 | } 33 | let mut poses = poses.map(|pose| pose.isometry().inverse()); 34 | let mut bests = [[f64::INFINITY; 2]; 2]; 35 | let mut no_improve_for = 0; 36 | for iteration in 0..iterations { 37 | // For the sums here, we use a VERY small number. 38 | // This is so that if the gradient is zero for every data point (we are 100% perfect), 39 | // then it will not turn the delta into NaN by taking the reciprocal of 0 (infinity) and multiplying 40 | // it by 0. 41 | let mut nets = [ 42 | (Se3TangentSpace::identity(), 0.0, 0.0), 43 | (Se3TangentSpace::identity(), 0.0, 0.0), 44 | ]; 45 | let tscale = poses 46 | .iter() 47 | .map(|pose| pose.translation.vector.norm()) 48 | .sum::(); 49 | for &observations in landmarks { 50 | let deltas = landmark_gradients(poses, observations); 51 | 52 | for ((l1sum, ts, rs), &delta) in nets.iter_mut().zip(deltas.iter()) { 53 | *ts += (delta.translation.norm() + tscale * epsilon).recip(); 54 | *rs += (delta.rotation.norm() + epsilon).recip(); 55 | *l1sum += delta.l1(); 56 | } 57 | } 58 | 59 | // Apply the harmonic mean as per the Weiszfeld algorithm from the paper 60 | // "Sur le point pour lequel la somme des distances de n points donn ́es est minimum." 61 | let mut deltas = [Se3TangentSpace::identity(); 2]; 62 | for (delta, (l1sum, ts, rs)) in deltas.iter_mut().zip(nets) { 63 | *delta = l1sum 64 | .scale(optimization_rate) 65 | .scale_translation(ts.recip()) 66 | .scale_rotation(rs.recip()); 67 | } 68 | 69 | no_improve_for += 1; 70 | for ([best_t, best_r], (l1, _, _)) in bests.iter_mut().zip(nets.iter()) { 71 | let t = l1.translation.norm(); 72 | let r = l1.rotation.norm(); 73 | if *best_t > t { 74 | *best_t = t; 75 | no_improve_for = 0; 76 | } 77 | if *best_r > r { 78 | *best_r = r; 79 | no_improve_for = 0; 80 | } 81 | } 82 | 83 | // Check if all of the optimizers reached stability. 84 | if no_improve_for >= 50 { 85 | log::info!( 86 | "terminating three-view optimization due to stabilizing on iteration {}", 87 | iteration 88 | ); 89 | log::info!( 90 | "first rotation magnitude: {}", 91 | nets[0].0.rotation.norm() / landmarks.len() as f64 92 | ); 93 | log::info!( 94 | "second rotation magnitude: {}", 95 | nets[1].0.rotation.norm() / landmarks.len() as f64 96 | ); 97 | break; 98 | } 99 | 100 | // Run everything through the optimizer and keep track of if all of them finished. 101 | for (pose, delta) in poses.iter_mut().zip(deltas.iter()) { 102 | // Perturb slightly using the previous delta to avoid getting stuck overlapping with a datapoint. 103 | // This can occur to the Weizfeld algorithm because when you overlap a datapoint perfectly, it produces a 0. 104 | // The 0 ends up causing the whole harmonic mean to go to 0. This small perturbation helps avoid 105 | // getting stuck in a local minima. 106 | *pose = delta.isometry() * *pose; 107 | } 108 | 109 | // If we are on the last iteration, print some logs indicating so. 110 | if iteration == iterations - 1 { 111 | log::info!("terminating three-view optimization due to reaching maximum iterations"); 112 | log::info!( 113 | "first rotation magnitude: {}", 114 | nets[0].0.rotation.norm() / landmarks.len() as f64 115 | ); 116 | log::info!( 117 | "second rotation magnitude: {}", 118 | nets[1].0.rotation.norm() / landmarks.len() as f64 119 | ); 120 | break; 121 | } 122 | } 123 | poses.map(|pose| pose.inverse().into()) 124 | } 125 | 126 | pub fn three_view_simple_optimize_l2( 127 | poses: [CameraToCamera; 2], 128 | optimization_rate: f64, 129 | iterations: usize, 130 | landmarks: &[[UnitVector3; 3]], 131 | ) -> [CameraToCamera; 2] { 132 | if landmarks.is_empty() { 133 | return poses; 134 | } 135 | let inv_landmark_len = (landmarks.len() as f64).recip(); 136 | let mut poses = poses.map(|pose| pose.isometry().inverse()); 137 | let mut bests = [[f64::INFINITY; 2]; 2]; 138 | let mut no_improve_for = 0; 139 | for iteration in 0..iterations { 140 | // Collect the sums of all the L2 distances. 141 | let mut nets = [Se3TangentSpace::identity(), Se3TangentSpace::identity()]; 142 | for &observations in landmarks { 143 | let deltas = landmark_gradients(poses, observations); 144 | 145 | for (l2sum, &delta) in nets.iter_mut().zip(deltas.iter()) { 146 | *l2sum += delta; 147 | } 148 | } 149 | 150 | // Compute the delta by applying the L2 distance with the inverse landmark length to 151 | // get the average length times the rate. 152 | let mut deltas = [Se3TangentSpace::identity(); 2]; 153 | for (delta, l2sum) in deltas.iter_mut().zip(nets) { 154 | *delta = l2sum.scale(inv_landmark_len * optimization_rate); 155 | } 156 | 157 | no_improve_for += 1; 158 | for ([best_t, best_r], l2sum) in bests.iter_mut().zip(nets.iter()) { 159 | let t = l2sum.translation.norm(); 160 | let r = l2sum.rotation.norm(); 161 | if *best_t > t { 162 | *best_t = t; 163 | no_improve_for = 0; 164 | } 165 | if *best_r > r { 166 | *best_r = r; 167 | no_improve_for = 0; 168 | } 169 | } 170 | 171 | // Check if all of the optimizers reached stability. 172 | if no_improve_for >= 50 { 173 | log::info!( 174 | "terminating three-view optimization due to stabilizing on iteration {}", 175 | iteration 176 | ); 177 | log::info!("first rotation magnitude: {}", deltas[0].rotation.norm()); 178 | log::info!("second rotation magnitude: {}", deltas[1].rotation.norm()); 179 | break; 180 | } 181 | 182 | // Run everything through the optimizer and keep track of if all of them finished. 183 | for (pose, delta) in poses.iter_mut().zip(deltas.iter()) { 184 | // Perturb slightly using the previous delta to avoid getting stuck overlapping with a datapoint. 185 | // This can occur to the Weizfeld algorithm because when you overlap a datapoint perfectly, it produces a 0. 186 | // The 0 ends up causing the whole harmonic mean to go to 0. This small perturbation helps avoid 187 | // getting stuck in a local minima. 188 | *pose = delta.isometry() * *pose; 189 | } 190 | 191 | // If we are on the last iteration, print some logs indicating so. 192 | if iteration == iterations - 1 { 193 | log::info!("terminating three-view optimization due to reaching maximum iterations"); 194 | log::info!("first rotation magnitude: {}", deltas[0].rotation.norm()); 195 | log::info!("second rotation magnitude: {}", deltas[1].rotation.norm()); 196 | break; 197 | } 198 | } 199 | poses.map(|pose| pose.inverse().into()) 200 | } 201 | 202 | /// Performs adaptive optimizations 203 | pub fn three_view_adaptive_optimize_l2( 204 | poses: [CameraToCamera; 2], 205 | iterations: usize, 206 | landmarks: &[[UnitVector3; 3]], 207 | ) -> [CameraToCamera; 2] { 208 | if landmarks.is_empty() { 209 | return poses; 210 | } 211 | let inv_landmark_len = (landmarks.len() as f64).recip(); 212 | let mut poses = poses.map(|pose| pose.isometry().inverse()); 213 | // let mut max_ema_variance = SVector::::zeros(); 214 | for iteration in 0..iterations { 215 | // For the sums here, we use a VERY small number. 216 | // This is so that if the gradient is zero for every data point (we are 100% perfect), 217 | // then it will not turn the delta into NaN by taking the reciprocal of 0 (infinity) and multiplying 218 | // it by 0. 219 | let mut nets = [ 220 | (Se3TangentSpace::identity(), 0.0, 0.0), 221 | (Se3TangentSpace::identity(), 0.0, 0.0), 222 | ]; 223 | for &observations in landmarks { 224 | let gradients = landmark_gradients(poses, observations); 225 | 226 | for ((l2sum, tv, rv), &delta) in nets.iter_mut().zip(gradients.iter()) { 227 | *l2sum += delta; 228 | *tv += delta.translation.norm(); 229 | *rv += delta.rotation.norm(); 230 | } 231 | } 232 | 233 | let mut gradients = [Se3TangentSpace::identity(); 2]; 234 | for (gradient, (l2sum, tv, rv)) in gradients.iter_mut().zip(nets) { 235 | // Correct for the number of things summed to form the mean and variance. 236 | let l2 = l2sum.scale(inv_landmark_len); 237 | let tstd = tv * inv_landmark_len; 238 | let rstd = rv * inv_landmark_len; 239 | // Scale by epsilon on the top and bottom ((mean + epsilon) / (sqrt(variance) + epsilon) 240 | // such that as mean and variance become very small that we approach a ratio of 1.0, 241 | // which will tend towards the optimization rate. This allows us to use epsilon to control 242 | // roughly how precise we want the result. 243 | let trate = l2.translation.norm() / tstd; 244 | let trate = if trate.is_finite() { trate } else { 0.0 }; 245 | let rrate = l2.rotation.norm() / rstd; 246 | let rrate = if rrate.is_finite() { rrate } else { 0.0 }; 247 | 248 | *gradient = l2.scale_translation(trate).scale_rotation(rrate); 249 | } 250 | 251 | // Update the poses using the delta. 252 | for (pose, delta) in poses.iter_mut().zip(&gradients) { 253 | *pose = delta.isometry() * *pose; 254 | } 255 | 256 | // If we are on the last iteration, print some logs indicating so. 257 | if iteration == iterations - 1 { 258 | log::info!("terminating three-view optimization due to reaching maximum iterations"); 259 | log::info!( 260 | "first rotation magnitude: {}", 261 | nets[0].0.rotation.norm() * inv_landmark_len 262 | ); 263 | log::info!( 264 | "second rotation magnitude: {}", 265 | nets[1].0.rotation.norm() * inv_landmark_len 266 | ); 267 | break; 268 | } 269 | } 270 | 271 | poses.map(|pose| pose.inverse().into()) 272 | } 273 | --------------------------------------------------------------------------------