Add basic optical flow implementation

finchie

Jul 16, 2023, 2:04 PM

2ZEXFAG3WLJUWQ6BLYODFO3OCTYAVAOZAQYYPMSZYEGCO4AQU5VAC

Dependencies

[2] S73DIQYD Scaffold project

Change contents

replacement in src/main.rs at line 1

[2.49]→[2.50:93](∅→∅)

fn main() {
println!("Hello, world!");

[2.49]

[2.93]

use opencv::core::{
cart_to_polar, merge, no_array, normalize, split, Mat as Matrix, MatTraitConst,
MatTraitConstManual, Ptr, Size, ToOutputArray, Vector, CV_32F, CV_8U, NORM_MINMAX,
};
use opencv::imgcodecs::imwrite;
use opencv::imgproc::{cvt_color, COLOR_HSV2BGR};
use opencv::optflow::{calc_optical_flow_dense_rlof, InterpolationType, RLOFOpticalFlowParameter};
use opencv::prelude::MatExprTraitConst;
use opencv::types::VectorOfMat;
use opencv::videoio::{
VideoCapture, VideoCaptureTrait, VideoCaptureTraitConst, VideoWriter, VideoWriterTrait,
CAP_ANY, CAP_PROP_FOURCC, CAP_PROP_FPS, CAP_PROP_FRAME_HEIGHT, CAP_PROP_FRAME_WIDTH,
};

fn read_frame(
video: &mut VideoCapture,
frame: &mut impl ToOutputArray,
) -> Result<bool, anyhow::Error> {
Ok(video.read(frame)?)
}

fn main() -> Result<(), anyhow::Error> {
let mut input_video = VideoCapture::from_file("../clips/edit.webm", CAP_ANY)?;
let mut output_video = VideoWriter::new(
"output.webm",
input_video.get(CAP_PROP_FOURCC)? as i32,
input_video.get(CAP_PROP_FPS)?,
Size::new(
input_video.get(CAP_PROP_FRAME_WIDTH)? as i32,
input_video.get(CAP_PROP_FRAME_HEIGHT)? as i32,
),
true,
)?;

let mut frame_count = 0;

let mut last_frame = Matrix::default();
input_video
.read(&mut last_frame)
.expect("Video cannot be empty");
let mut current_frame = Matrix::default();

while read_frame(&mut input_video, &mut current_frame)? {
println!("Frame {frame_count}");
let mut computed_flow = unsafe { Matrix::new_size(current_frame.size()?, CV_32F)? };
calc_optical_flow_dense_rlof(
&last_frame,
&current_frame,
&mut computed_flow,
Ptr::new(RLOFOpticalFlowParameter::default()?),
1 as f32,
Size::new(6, 6),
InterpolationType::INTERP_EPIC,
128,
0.005,
999 as f32,
15,
100,
true,
500 as f32,
1.5,
false,
)?;

let mut flow_parts: VectorOfMat = Vec::with_capacity(2).into();
split(&computed_flow, &mut flow_parts)?;
assert_eq!(flow_parts.len(), 2);

let mut magnitude = Matrix::default();
let mut normalized_magnitude = Matrix::default();
let mut angle = Matrix::default();
cart_to_polar(
&flow_parts.get(0)?,
&flow_parts.get(1)?,
&mut magnitude,
&mut angle,
true,
)?;
normalize(
&magnitude,
&mut normalized_magnitude,
0 as f64,
1 as f64,
NORM_MINMAX,
-1,
&no_array(),
)?;
angle = (angle * (1_f64 / 360_f64) * (180_f64 / 255_f64))
.into_result()?
.to_mat()?;

let mut hsv_vec = VectorOfMat::with_capacity(3);
hsv_vec.push(angle.clone());
hsv_vec.push(Matrix::ones_size(angle.size()?, CV_32F)?.to_mat()?);
hsv_vec.push(normalized_magnitude);
assert_eq!(hsv_vec.len(), 3);

let mut hsv = Matrix::default();
let mut hsv_8 = Matrix::default();
let mut bgr = Matrix::default();

merge(&hsv_vec, &mut hsv)?;
hsv.convert_to(&mut hsv_8, CV_8U, 255 as f64, 0 as f64)?;
cvt_color(&hsv_8, &mut bgr, COLOR_HSV2BGR, 0)?;
// output_video.write(&bgr)?;
imwrite(
&format!("frames/frame{frame_count}.jpg"),
&bgr,
&Vector::new(),
)?;

assert_eq!(current_frame.size()?, last_frame.size()?);
last_frame = current_frame.clone();

// if frame_count > 5 {
// break;
// } else {
// }
frame_count += 1;
}

Ok(())
edit in Cargo.toml at line 12

[2.2097]

anyhow = { version = "1.0.71", features = ["backtrace"] }
opencv = { version = "0.83.0", default-features = false, features = ["optflow", "videoio", "imgproc", "imgcodecs"] }
edit in Cargo.lock at line 4

[2.2237]

[2.2237]

[[package]]
name = "addr2line"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
dependencies = [
"gimli",
]

[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

[[package]]
name = "aho-corasick"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
dependencies = [
"memchr",
]

[[package]]
name = "anyhow"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
dependencies = [
"backtrace",
]

[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

[[package]]
name = "backtrace"
version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]

[[package]]
name = "cc"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
dependencies = [
"jobserver",
]

[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
edit in Cargo.lock at line 75

[2.2250]

[2.2250]

name = "clang"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c044c781163c001b913cd018fc95a628c50d0d2dfea8bca77dad71edb16e37"
dependencies = [
"clang-sys",
"libc",
]

[[package]]
name = "clang-sys"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
dependencies = [
"glob",
"libc",
]

[[package]]
edit in Cargo.lock at line 97

[2.2283]

dependencies = [
"anyhow",
"opencv",
]

[[package]]
name = "dunce"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"

[[package]]
name = "gimli"
version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"

[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"

[[package]]
name = "jobserver"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
dependencies = [
"libc",
]

[[package]]
name = "libc"
version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"

[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"

[[package]]
name = "miniz_oxide"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
dependencies = [
"adler",
]

[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]

[[package]]
name = "object"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
dependencies = [
"memchr",
]

[[package]]
name = "once_cell"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"

[[package]]
name = "opencv"
version = "0.83.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "998598d7574b0ec0f7230b7b9782ab516ccf612dbacb6ba83ca1b0e2af537359"
dependencies = [
"cc",
"dunce",
"jobserver",
"libc",
"num-traits",
"once_cell",
"opencv-binding-generator",
"pkg-config",
"semver",
"shlex",
"vcpkg",
]

[[package]]
name = "opencv-binding-generator"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb6b3b79658fe7db11c250913b85915b4cadc7bf5f70181572f55437327fb294"
dependencies = [
"clang",
"clang-sys",
"dunce",
"once_cell",
"percent-encoding",
"regex",
]

[[package]]
name = "percent-encoding"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"

[[package]]
name = "pkg-config"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"

[[package]]
name = "regex"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]

[[package]]
name = "regex-automata"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]

[[package]]
name = "regex-syntax"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"

[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"

[[package]]
name = "semver"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"

[[package]]
name = "shlex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"

[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
replacement in .ignore at line 3

[2.2332]→[2.2332:2350](∅→∅)

# Added by cargo

[2.2332]

[2.2350]

/frames
edit in .ignore at line 5

[2.2358]

output.webm