finchie/cutup - Change 2ZEXFAG3WLJUWQ6BLYODFO3OCTYAVAOZAQYYPMSZYEGCO4AQU5VAC

Add basic optical flow implementation

This is currently horrendously slow, as it operates on the full-resolution image. Also the maths is probaly wrong, and compressed videos are handled very poorly.

Created by finchie on July 16, 2023

2ZEXFAG3WLJUWQ6BLYODFO3OCTYAVAOZAQYYPMSZYEGCO4AQU5VAC

Dependencies

[2] S73DIQYDIVAHCLUJZZDOL6YT62OFVZJQYJ2F3F43APJFOOIN5WTAC

In channels

main

Change contents

Replacement in src/main.rs at line 1 [2.49]

B:BD[2.49] → [2.50:93]

fn main() {
    println!("Hello, world!");

[2.49]

[2.93]

use opencv::core::{
    cart_to_polar, merge, no_array, normalize, split, Mat as Matrix, MatTraitConst,
    MatTraitConstManual, Ptr, Size, ToOutputArray, Vector, CV_32F, CV_8U, NORM_MINMAX,
};
use opencv::imgcodecs::imwrite;
use opencv::imgproc::{cvt_color, COLOR_HSV2BGR};
use opencv::optflow::{calc_optical_flow_dense_rlof, InterpolationType, RLOFOpticalFlowParameter};
use opencv::prelude::MatExprTraitConst;
use opencv::types::VectorOfMat;
use opencv::videoio::{
    VideoCapture, VideoCaptureTrait, VideoCaptureTraitConst, VideoWriter, VideoWriterTrait,
    CAP_ANY, CAP_PROP_FOURCC, CAP_PROP_FPS, CAP_PROP_FRAME_HEIGHT, CAP_PROP_FRAME_WIDTH,
};
fn read_frame(
    video: &mut VideoCapture,
    frame: &mut impl ToOutputArray,
) -> Result<bool, anyhow::Error> {
    Ok(video.read(frame)?)
}
fn main() -> Result<(), anyhow::Error> {
    let mut input_video = VideoCapture::from_file("../clips/edit.webm", CAP_ANY)?;
    let mut output_video = VideoWriter::new(
        "output.webm",
        input_video.get(CAP_PROP_FOURCC)? as i32,
        input_video.get(CAP_PROP_FPS)?,
        Size::new(
            input_video.get(CAP_PROP_FRAME_WIDTH)? as i32,
            input_video.get(CAP_PROP_FRAME_HEIGHT)? as i32,
        ),
        true,
    )?;
    let mut frame_count = 0;
    let mut last_frame = Matrix::default();
    input_video
        .read(&mut last_frame)
        .expect("Video cannot be empty");
    let mut current_frame = Matrix::default();
    while read_frame(&mut input_video, &mut current_frame)? {
        println!("Frame {frame_count}");
        let mut computed_flow = unsafe { Matrix::new_size(current_frame.size()?, CV_32F)? };
        calc_optical_flow_dense_rlof(
            &last_frame,
            &current_frame,
            &mut computed_flow,
            Ptr::new(RLOFOpticalFlowParameter::default()?),
            1 as f32,
            Size::new(6, 6),
            InterpolationType::INTERP_EPIC,
            128,
            0.005,
            999 as f32,
            15,
            100,
            true,
            500 as f32,
            1.5,
            false,
        )?;
        let mut flow_parts: VectorOfMat = Vec::with_capacity(2).into();
        split(&computed_flow, &mut flow_parts)?;
        assert_eq!(flow_parts.len(), 2);
        let mut magnitude = Matrix::default();
        let mut normalized_magnitude = Matrix::default();
        let mut angle = Matrix::default();
        cart_to_polar(
            &flow_parts.get(0)?,
            &flow_parts.get(1)?,
            &mut magnitude,
            &mut angle,
            true,
        )?;
        normalize(
            &magnitude,
            &mut normalized_magnitude,
            0 as f64,
            1 as f64,
            NORM_MINMAX,
            -1,
            &no_array(),
        )?;
        angle = (angle * (1_f64 / 360_f64) * (180_f64 / 255_f64))
            .into_result()?
            .to_mat()?;
        let mut hsv_vec = VectorOfMat::with_capacity(3);
        hsv_vec.push(angle.clone());
        hsv_vec.push(Matrix::ones_size(angle.size()?, CV_32F)?.to_mat()?);
        hsv_vec.push(normalized_magnitude);
        assert_eq!(hsv_vec.len(), 3);
        let mut hsv = Matrix::default();
        let mut hsv_8 = Matrix::default();
        let mut bgr = Matrix::default();
        merge(&hsv_vec, &mut hsv)?;
        hsv.convert_to(&mut hsv_8, CV_8U, 255 as f64, 0 as f64)?;
        cvt_color(&hsv_8, &mut bgr, COLOR_HSV2BGR, 0)?;
        // output_video.write(&bgr)?;
        imwrite(
            &format!("frames/frame{frame_count}.jpg"),
            &bgr,
            &Vector::new(),
        )?;
        assert_eq!(current_frame.size()?, last_frame.size()?);
        last_frame = current_frame.clone();
        // if frame_count > 5 {
        //     break;
        // } else {
        // }
        frame_count += 1;
    }
    Ok(())

Insertion in Cargo.toml at line 12 [2.1793]

[2.2097]

anyhow = { version = "1.0.71", features = ["backtrace"] }
opencv = { version = "0.83.0", default-features = false, features = ["optflow", "videoio", "imgproc", "imgcodecs"] }

Insertion in Cargo.lock at line 4 [2.2133]

[2.2237]


[[package]]
name = "addr2line"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
dependencies = [
 "gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
dependencies = [
 "memchr",
]
[[package]]
name = "anyhow"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
dependencies = [
 "backtrace",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
dependencies = [
 "addr2line",
 "cc",
 "cfg-if",
 "libc",
 "miniz_oxide",
 "object",
 "rustc-demangle",
]
[[package]]
name = "cc"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
dependencies = [
 "jobserver",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

Insertion in Cargo.lock at line 75 [2.2133]

[2.2250]

name = "clang"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c044c781163c001b913cd018fc95a628c50d0d2dfea8bca77dad71edb16e37"
dependencies = [
 "clang-sys",
 "libc",
]
[[package]]
name = "clang-sys"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
dependencies = [
 "glob",
 "libc",
]
[[package]]

Insertion in Cargo.lock at line 97 [2.2133]

[2.2283]

dependencies = [
 "anyhow",
 "opencv",
]
[[package]]
name = "dunce"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
[[package]]
name = "gimli"
version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "jobserver"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
dependencies = [
 "libc",
]
[[package]]
name = "libc"
version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "miniz_oxide"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
dependencies = [
 "adler",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
 "autocfg",
]
[[package]]
name = "object"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
dependencies = [
 "memchr",
]
[[package]]
name = "once_cell"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "opencv"
version = "0.83.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "998598d7574b0ec0f7230b7b9782ab516ccf612dbacb6ba83ca1b0e2af537359"
dependencies = [
 "cc",
 "dunce",
 "jobserver",
 "libc",
 "num-traits",
 "once_cell",
 "opencv-binding-generator",
 "pkg-config",
 "semver",
 "shlex",
 "vcpkg",
]
[[package]]
name = "opencv-binding-generator"
version = "0.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb6b3b79658fe7db11c250913b85915b4cadc7bf5f70181572f55437327fb294"
dependencies = [
 "clang",
 "clang-sys",
 "dunce",
 "once_cell",
 "percent-encoding",
 "regex",
]
[[package]]
name = "percent-encoding"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]]
name = "pkg-config"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "regex"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
dependencies = [
 "aho-corasick",
 "memchr",
 "regex-automata",
 "regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
dependencies = [
 "aho-corasick",
 "memchr",
 "regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "semver"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
[[package]]
name = "shlex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"

Replacement in .ignore at line 3 [2.2316]
B:BD[2.2332] → [2.2332:2350]
```
# Added by cargo
```
[2.2332]
[2.2350]
```
/frames
```
Insertion in .ignore at line 5 [2.2316]
[2.2358]
```
output.webm
```