Skip to content

Commit 23962ec

Browse files
powei-linpowei-lin
authored
Add fast remap (#6)
* bench * remove par iter * save * add doc * fast remap * typo * typo * fast rgb * bench * readme * update version * panic
1 parent 46ddbfe commit 23962ec

File tree

7 files changed

+266
-7
lines changed

7 files changed

+266
-7
lines changed

‎Cargo.toml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "camera-intrinsic-model"
3-
version = "0.4.1"
3+
version = "0.5.0"
44
edition = "2021"
55
authors = ["Powei Lin <[email protected]>"]
66
readme = "README.md"
@@ -10,7 +10,7 @@ homepage = "https://github.com/powei-lin/camera-intrinsic-model-rs"
1010
repository = "https://github.com/powei-lin/camera-intrinsic-model-rs"
1111
keywords = ["camera-intrinsic", "intrinsic", "fisheye"]
1212
categories = ["data-structures", "science", "mathematics", "science::robotics"]
13-
exclude = ["/.github/*", "*.ipynb", "./scripts/*", "examples/*", "tests/*", "data/*"]
13+
exclude = ["/.github/*", "*.ipynb", "./scripts/*", "examples/*", "tests/*", "data/*", "benches/*"]
1414

1515
[dependencies]
1616
image = "0.25.6"
@@ -25,6 +25,11 @@ name = "remap"
2525
[[example]]
2626
name = "stereo_rectify"
2727

28+
[[bench]]
29+
name = "bench"
30+
harness = false
31+
2832
[dev-dependencies]
33+
diol = "0.13.0"
2934
imageproc = "0.25.0"
3035
rand = "0.9.0"

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@ cargo run -r --example remap
2525
cargo run -r --example stereo_rectify
2626
```
2727

28+
## Benchmark
29+
Remapping to 1024x1024 on m4 mac mini.
30+
```
31+
╭───────────────────────────────────────────────────────────────────────╮
32+
│ remap │
33+
├────────────────┬──────┬───────────┬───────────┬───────────┬───────────┤
34+
│ benchmark │ args │ fastest │ median │ mean │ stddev │
35+
├────────────────┼──────┼───────────┼───────────┼───────────┼───────────┤
36+
│ mono8 normal │ None │ 732.17 µs │ 827.00 µs │ 858.60 µs │ 94.88 µs │
37+
│ mono8 fast │ None │ 272.00 µs │ 342.25 µs │ 360.68 µs │ 228.13 µs │
38+
│ rgb8 normal │ None │ 1.87 ms │ 2.00 ms │ 2.04 ms │ 143.60 µs │
39+
│ rgb8 fast │ None │ 751.67 µs │ 824.54 µs │ 851.30 µs │ 79.45 µs │
40+
╰────────────────┴──────┴───────────┴───────────┴───────────┴───────────╯
41+
```
42+
2843
## Acknowledgements
2944
Links:
3045
* https://cvg.cit.tum.de/data/datasets/visual-inertial-dataset

‎benches/bench.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
use camera_intrinsic_model::{compute_for_fast_remap, fast_remap, model_from_json, remap};
2+
use diol::prelude::*;
3+
use image::{DynamicImage, ImageReader};
4+
5+
fn main() -> eyre::Result<()> {
6+
let bench = Bench::from_args()?;
7+
bench.register_many(
8+
"remap",
9+
list![
10+
bench_remap.with_name("mono8 normal"),
11+
bench_remap_fast.with_name("mono8 fast"),
12+
bench_remap_rgb.with_name("rgb8 normal"),
13+
bench_remap_fast_rgb.with_name("rgb8 fast"),
14+
],
15+
[None],
16+
);
17+
bench.run()?;
18+
Ok(())
19+
}
20+
21+
fn bench_remap(bencher: Bencher, _dummy: Option<bool>) {
22+
let model1 = model_from_json("data/eucm0.json");
23+
let new_w_h = 1024;
24+
let img = ImageReader::open("data/tum_vi_with_chart.png")
25+
.unwrap()
26+
.decode()
27+
.unwrap();
28+
let p = model1.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
29+
let (xmap, ymap) = model1.init_undistort_map(&p, (new_w_h, new_w_h), None);
30+
let img_l8 = DynamicImage::ImageLuma8(img.to_luma8());
31+
bencher.bench(|| {
32+
let _ = remap(&img_l8, &xmap, &ymap);
33+
});
34+
}
35+
36+
fn bench_remap_rgb(bencher: Bencher, _dummy: Option<bool>) {
37+
let model1 = model_from_json("data/eucm0.json");
38+
let new_w_h = 1024;
39+
let img = ImageReader::open("data/tum_vi_with_chart.png")
40+
.unwrap()
41+
.decode()
42+
.unwrap();
43+
let p = model1.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
44+
let (xmap, ymap) = model1.init_undistort_map(&p, (new_w_h, new_w_h), None);
45+
let img_rgb8 = DynamicImage::ImageRgb8(img.to_rgb8());
46+
bencher.bench(|| {
47+
let _ = remap(&img_rgb8, &xmap, &ymap);
48+
});
49+
}
50+
51+
fn bench_remap_fast(bencher: Bencher, _dummy: Option<bool>) {
52+
let model1 = model_from_json("data/eucm0.json");
53+
let new_w_h = 1024;
54+
let img = ImageReader::open("data/tum_vi_with_chart.png")
55+
.unwrap()
56+
.decode()
57+
.unwrap();
58+
let p = model1.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
59+
let (xmap, ymap) = model1.init_undistort_map(&p, (new_w_h, new_w_h), None);
60+
let img_l8 = DynamicImage::ImageLuma8(img.to_luma8());
61+
let xy_pos_weight = compute_for_fast_remap(&xmap, &ymap);
62+
bencher.bench(|| {
63+
let _ = fast_remap(&img_l8, (new_w_h, new_w_h), &xy_pos_weight);
64+
});
65+
}
66+
67+
fn bench_remap_fast_rgb(bencher: Bencher, _dummy: Option<bool>) {
68+
let model1 = model_from_json("data/eucm0.json");
69+
let new_w_h = 1024;
70+
let img = ImageReader::open("data/tum_vi_with_chart.png")
71+
.unwrap()
72+
.decode()
73+
.unwrap();
74+
let p = model1.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
75+
let (xmap, ymap) = model1.init_undistort_map(&p, (new_w_h, new_w_h), None);
76+
let img_rgb = DynamicImage::ImageRgb8(img.to_rgb8());
77+
let xy_pos_weight = compute_for_fast_remap(&xmap, &ymap);
78+
bencher.bench(|| {
79+
let _ = fast_remap(&img_rgb, (new_w_h, new_w_h), &xy_pos_weight);
80+
});
81+
}

‎benches/opencv_test.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import cv2
2+
import numpy as np
3+
from time import perf_counter
4+
5+
6+
def main():
7+
img = (255 * np.random.random((512, 512))).astype(np.uint8)
8+
mapx = (511 * np.random.random((1024, 1024))).astype(np.float32)
9+
mapy = (511 * np.random.random((1024, 1024))).astype(np.float32)
10+
# mapx, mapy = cv2.convertMaps(mapx, mapy, cv2.CV_16SC2)
11+
tt = 1000
12+
t0 = perf_counter()
13+
for i in range(tt):
14+
cv2.remap(img, mapx, mapy, cv2.INTER_LINEAR)
15+
t1 = perf_counter()
16+
print(f"{(t1 - t0) / tt}")
17+
18+
19+
if __name__ == "__main__":
20+
main()

‎examples/remap.rs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use camera_intrinsic_model::*;
2-
use image::ImageReader;
2+
use image::{DynamicImage, ImageReader};
33
use nalgebra as na;
44

55
fn main() {
@@ -22,6 +22,15 @@ fn main() {
2222
let new_w_h = 1024;
2323
let p = model1.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
2424
let (xmap, ymap) = model1.init_undistort_map(&p, (new_w_h, new_w_h), None);
25-
let remaped = remap(&img, &xmap, &ymap);
26-
remaped.save("remaped.png").unwrap()
25+
let img_l8 = DynamicImage::ImageLuma8(img.to_luma8());
26+
let remaped = remap(&img_l8, &xmap, &ymap);
27+
remaped.save("remaped0.png").unwrap();
28+
29+
let xy_pos_weight = compute_for_fast_remap(&xmap, &ymap);
30+
let remaped1 = fast_remap(&img_l8, (new_w_h, new_w_h), &xy_pos_weight);
31+
remaped1.save("remaped1.png").unwrap();
32+
33+
let img_rgb8 = DynamicImage::ImageRgb8(img.to_rgb8());
34+
let remaped1 = fast_remap(&img_rgb8, (new_w_h, new_w_h), &xy_pos_weight);
35+
remaped1.save("remaped2.png").unwrap();
2736
}

‎src/generic_functions.rs‎

Lines changed: 120 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use super::generic_model::*;
22

3+
use image::{DynamicImage, GenericImageView, GrayImage, ImageBuffer};
34
use nalgebra as na;
45
use rayon::prelude::*;
56

@@ -44,7 +45,7 @@ pub fn init_undistort_map(
4445
.into_par_iter()
4546
.flat_map(|y| {
4647
(0..new_w_h.0)
47-
.into_par_iter()
48+
.into_iter()
4849
.map(|x| {
4950
rmat_inv * na::Vector3::new((x as f64 - cx) / fx, (y as f64 - cy) / fy, 1.0)
5051
})
@@ -53,7 +54,7 @@ pub fn init_undistort_map(
5354
.collect();
5455
let p2ds = camera_model.project(&p3ds);
5556
let (xvec, yvec): (Vec<f32>, Vec<f32>) = p2ds
56-
.par_iter()
57+
.iter()
5758
.map(|xy| {
5859
if let Some(xy) = xy {
5960
(xy[0] as f32, xy[1] as f32)
@@ -67,6 +68,123 @@ pub fn init_undistort_map(
6768
(xmap, ymap)
6869
}
6970

71+
#[inline]
72+
fn interpolate_bilinear_weight(x: f32, y: f32) -> (u32, u32) {
73+
if x < 0.0 || x > 65535.0 {
74+
panic!("x not in [0-65535]");
75+
}
76+
if y < 0.0 || y > 65535.0 {
77+
panic!("y not in [0-65535]");
78+
}
79+
const UPPER: f32 = u8::MAX as f32;
80+
let x_weight = (UPPER * (x.ceil() - x)) as u32;
81+
let y_weight = (UPPER * (y.ceil() - y)) as u32;
82+
// 0-255
83+
(x_weight, y_weight)
84+
}
85+
86+
pub fn compute_for_fast_remap(
87+
xmap: &na::DMatrix<f32>,
88+
ymap: &na::DMatrix<f32>,
89+
) -> Vec<(u32, u32, u32, u32)> {
90+
let xy_pos_weight_vec: Vec<_> = xmap
91+
.iter()
92+
.zip(ymap.iter())
93+
.map(|(&x, &y)| {
94+
let (xw, yw) = interpolate_bilinear_weight(x, y);
95+
let x0 = x.floor() as u32;
96+
let y0 = y.floor() as u32;
97+
(x0, y0, xw, yw)
98+
})
99+
.collect();
100+
xy_pos_weight_vec
101+
}
102+
103+
fn reinterpret_vec(input: Vec<[u8; 3]>) -> Vec<u8> {
104+
let len = input.len() * 3;
105+
let capacity = input.capacity() * 3;
106+
let ptr = input.as_ptr() as *mut u8;
107+
std::mem::forget(input); // prevent dropping original vec
108+
unsafe { Vec::from_raw_parts(ptr, len, capacity) }
109+
}
110+
111+
pub fn fast_remap(
112+
img: &DynamicImage,
113+
new_w_h: (u32, u32),
114+
xy_pos_weight_vec: &[(u32, u32, u32, u32)],
115+
) -> DynamicImage {
116+
let remaped = match img {
117+
DynamicImage::ImageLuma8(image_buffer) => {
118+
let val: Vec<u8> = xy_pos_weight_vec
119+
.par_iter()
120+
.map(|&(x0, y0, xw0, yw0)| {
121+
let p00 = unsafe { image_buffer.unsafe_get_pixel(x0, y0)[0] as u32 };
122+
let p10 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0)[0] as u32 };
123+
let p01 = unsafe { image_buffer.unsafe_get_pixel(x0, y0 + 1)[0] as u32 };
124+
let p11 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0 + 1)[0] as u32 };
125+
let xw1 = 255 - xw0;
126+
let yw1 = 255 - yw0;
127+
const UPPER_UPPER: u32 = 255 * 255;
128+
let p =
129+
((p00 * xw0 * yw0 + p10 * xw1 * yw0 + p01 * xw0 * yw1 + p11 * xw1 * yw1)
130+
/ UPPER_UPPER) as u8;
131+
p
132+
})
133+
.collect();
134+
let img = GrayImage::from_vec(new_w_h.0, new_w_h.1, val).unwrap();
135+
DynamicImage::ImageLuma8(img)
136+
}
137+
DynamicImage::ImageRgb8(image_buffer) => {
138+
let val: Vec<[u8; 3]> = xy_pos_weight_vec
139+
.par_iter()
140+
.map(|&(x0, y0, xw0, yw0)| {
141+
let p00 = unsafe { image_buffer.unsafe_get_pixel(x0, y0) };
142+
let p10 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0) };
143+
let p01 = unsafe { image_buffer.unsafe_get_pixel(x0, y0 + 1) };
144+
let p11 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0 + 1) };
145+
let mut v = [0, 1, 2];
146+
v.iter_mut().for_each(|i| {
147+
let xw1 = 255 - xw0;
148+
let yw1 = 255 - yw0;
149+
let c = *i as usize;
150+
const UPPER_UPPER: u32 = 255 * 255;
151+
*i = ((p00.0[c] as u32 * xw0 * yw0
152+
+ p10.0[c] as u32 * xw1 * yw0
153+
+ p01.0[c] as u32 * xw0 * yw1
154+
+ p11.0[c] as u32 * xw1 * yw1)
155+
/ UPPER_UPPER) as u8;
156+
});
157+
v
158+
})
159+
.collect();
160+
let img = ImageBuffer::from_vec(new_w_h.0, new_w_h.1, reinterpret_vec(val)).unwrap();
161+
DynamicImage::ImageRgb8(img)
162+
}
163+
DynamicImage::ImageLuma16(image_buffer) => {
164+
let val: Vec<u16> = xy_pos_weight_vec
165+
.par_iter()
166+
.map(|&(x0, y0, xw0, yw0)| {
167+
let p00 = unsafe { image_buffer.unsafe_get_pixel(x0, y0)[0] as u32 };
168+
let p10 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0)[0] as u32 };
169+
let p01 = unsafe { image_buffer.unsafe_get_pixel(x0, y0 + 1)[0] as u32 };
170+
let p11 = unsafe { image_buffer.unsafe_get_pixel(x0 + 1, y0 + 1)[0] as u32 };
171+
let xw1 = 255 - xw0;
172+
let yw1 = 255 - yw0;
173+
const UPPER_UPPER: u32 = 255 * 255;
174+
let p =
175+
((p00 * xw0 * yw0 + p10 * xw1 * yw0 + p01 * xw0 * yw1 + p11 * xw1 * yw1)
176+
/ UPPER_UPPER) as u16;
177+
p
178+
})
179+
.collect();
180+
let img = ImageBuffer::from_vec(new_w_h.0, new_w_h.1, val).unwrap();
181+
DynamicImage::ImageLuma16(img)
182+
}
183+
_ => panic!("Only mono8, mono16, and rgb8 support fast remap."),
184+
};
185+
remaped
186+
}
187+
70188
/// Returns xmap and ymap for remaping
71189
///
72190
/// # Arguments

‎src/lib.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
//! camera-intrinsic-model is a library for distort/undistort images.
2+
//! # Examples
3+
//!
4+
//! ```
5+
//! use camera_intrinsic_model::*;
6+
//! let model = model_from_json("data/eucm0.json");
7+
//! let new_w_h = 1024;
8+
//! let p = model.estimate_new_camera_matrix_for_undistort(0.0, Some((new_w_h, new_w_h)));
9+
//! let (xmap, ymap) = model.init_undistort_map(&p, (new_w_h, new_w_h), None);
10+
//! // let remaped = remap(&img, &xmap, &ymap);
11+
//! ```
112
pub mod generic_functions;
213
pub mod generic_model;
314
pub mod io;

0 commit comments

Comments
 (0)