From a82c2a451b1c429a7fd283cdfe89be8087f5e8c5 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 26 Mar 2025 21:34:21 +0100 Subject: [PATCH 01/26] Initial support for dav1d and rav1e --- Cargo.lock | 275 +++++++++++++++++++++++++- Cargo.toml | 2 + node-hub/dora-av1-encoder/Cargo.toml | 15 ++ node-hub/dora-av1-encoder/example.py | 22 +++ node-hub/dora-av1-encoder/src/main.rs | 217 ++++++++++++++++++++ node-hub/dora-dav1d/Cargo.toml | 13 ++ node-hub/dora-dav1d/dataflow.yml | 53 +++++ node-hub/dora-dav1d/src/main.rs | 189 ++++++++++++++++++ 8 files changed, 784 insertions(+), 2 deletions(-) create mode 100644 node-hub/dora-av1-encoder/Cargo.toml create mode 100644 node-hub/dora-av1-encoder/example.py create mode 100644 node-hub/dora-av1-encoder/src/main.rs create mode 100644 node-hub/dora-dav1d/Cargo.toml create mode 100644 node-hub/dora-dav1d/dataflow.yml create mode 100644 node-hub/dora-dav1d/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 957cd526..9482a998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -273,6 +273,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "anstream" version = "0.6.18" @@ -1297,6 +1306,36 @@ dependencies = [ "num-traits", ] +[[package]] +name = "av-metrics" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "996ce95bbdb0203e5b91d4a0c9b81c0d67d11c80f884482a0c1ea19e732e3530" +dependencies = [ + "crossbeam", + "itertools 0.10.5", + "lab", + "num-traits", + "rayon", + "thiserror 1.0.66", + "v_frame", +] + +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "serde", + "v_frame", +] + [[package]] name = "axum" version = "0.7.9" @@ -1605,6 +1644,15 @@ dependencies = [ "sysinfo 0.33.1", ] +[[package]] +name = "built" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a6c0b39c38fd754ac338b00a88066436389c0f029da5d37d1e01091d9b7c17" +dependencies = [ + "git2", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -1905,6 +1953,16 @@ dependencies = [ "uuid 1.16.0", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -1986,6 +2044,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + [[package]] name = "clap" version = "3.2.25" @@ -2000,7 +2073,7 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap", + "textwrap 0.16.1", ] [[package]] @@ -2026,6 +2099,15 @@ dependencies = [ "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06f5378ea264ad4f82bbc826628b5aad714a75abf6ece087e923010eb937fb6" +dependencies = [ + "clap 4.5.20", +] + [[package]] name = "clap_derive" version = "3.2.25" @@ -2937,6 +3019,15 @@ dependencies = [ "num", ] +[[package]] +name = "dora-av1-encoder" +version = "0.3.10" +dependencies = [ + "dora-node-api", + "eyre", + "rav1e", +] + [[package]] name = "dora-cli" version = "0.3.10" @@ -3049,6 +3140,18 @@ dependencies = [ "zenoh 1.3.0", ] +[[package]] +name = "dora-dav1d" +version = "0.0.0" +dependencies = [ + "bitstream-io", + "dav1d", + "dora-node-api", + "eyre", + "log", + "structopt", +] + [[package]] name = "dora-download" version = "0.3.10" @@ -4086,6 +4189,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "fern" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c14694cbd524c8720dd69b0e3179344f04ebb5f90f2e4a440c6ea3b2f1ee" +dependencies = [ + "log", +] + [[package]] name = "ffmpeg-sidecar" version = "2.0.5" @@ -6044,6 +6156,12 @@ dependencies = [ "log", ] +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + [[package]] name = "lazy_static" version = "1.5.0" @@ -6135,6 +6253,16 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libgit2-sys" version = "0.16.2+1.7.2" @@ -6407,6 +6535,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if 1.0.0", + "rayon", +] + [[package]] name = "md5" version = "0.7.0" @@ -6918,6 +7056,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "nasm-rs" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4d98d0065f4b1daf164b3eafb11974c94662e5e2396cf03f32d0bb5c17da51" +dependencies = [ + "rayon", +] + [[package]] name = "nasm-rs" version = "0.3.0" @@ -7042,6 +7189,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "newline-converter" version = "0.2.2" @@ -9092,6 +9245,55 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-metrics", + "av1-grain", + "bitstream-io", + "built", + "cc", + "cfg-if 1.0.0", + "clap 4.5.20", + "clap_complete", + "console", + "fern", + "interpolate_name", + "itertools 0.12.1", + "ivf", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "nasm-rs 0.2.5", + "new_debug_unreachable", + "nom", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "scan_fmt", + "serde", + "serde-big-array", + "signal-hook", + "simd_helpers", + "system-deps", + "thiserror 1.0.66", + "toml", + "v_frame", + "y4m", +] + [[package]] name = "raw-cpuid" version = "10.7.0" @@ -9829,7 +10031,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "nasm-rs", + "nasm-rs 0.3.0", "parking_lot", "paste", "raw-cpuid 11.5.0", @@ -11591,6 +11793,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scan_fmt" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248" + [[package]] name = "schannel" version = "0.1.27" @@ -12183,6 +12391,15 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -12548,6 +12765,12 @@ dependencies = [ "float-cmp", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.10.0" @@ -12560,6 +12783,30 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "strum" version = "0.26.3" @@ -13897,6 +14144,18 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "serde", + "wasm-bindgen", +] + [[package]] name = "validated_struct" version = "2.1.1" @@ -13968,6 +14227,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -15404,6 +15669,12 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index f67b0fcb..50bbee92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,8 @@ members = [ "node-hub/dora-kit-car", "node-hub/dora-object-to-pose", "node-hub/dora-mistral-rs", + "node-hub/dora-av1-encoder", + "node-hub/dora-dav1d", "libraries/extensions/ros2-bridge", "libraries/extensions/ros2-bridge/msg-gen", "libraries/extensions/ros2-bridge/python", diff --git a/node-hub/dora-av1-encoder/Cargo.toml b/node-hub/dora-av1-encoder/Cargo.toml new file mode 100644 index 00000000..b0ee3d8a --- /dev/null +++ b/node-hub/dora-av1-encoder/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dora-av1-encoder" +edition = "2021" +version.workspace = true +description.workspace = true +documentation.workspace = true +license.workspace = true +repository.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rav1e = { version = "0.7.1", features = ["serialize"] } +dora-node-api = { workspace = true, features = ["tracing"] } +eyre = "0.6.8" diff --git a/node-hub/dora-av1-encoder/example.py b/node-hub/dora-av1-encoder/example.py new file mode 100644 index 00000000..80d54b89 --- /dev/null +++ b/node-hub/dora-av1-encoder/example.py @@ -0,0 +1,22 @@ +import av +import cv2 +import numpy as np + +# Open the AV1 video file +container = av.open("video.av1") + +# Get the video stream +stream = next(s for s in container.streams if s.type == "video") + +# Iterate over the video frames +for frame in container.decode(stream): + # Convert the frame to a NumPy array + img = frame.to_ndarray(format="bgr24") + + # Display the frame + cv2.imshow("Frame", img) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + +# Release the resources +cv2.destroyAllWindows() diff --git a/node-hub/dora-av1-encoder/src/main.rs b/node-hub/dora-av1-encoder/src/main.rs new file mode 100644 index 00000000..dbad450e --- /dev/null +++ b/node-hub/dora-av1-encoder/src/main.rs @@ -0,0 +1,217 @@ +// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved +// +// This source code is subject to the terms of the BSD 2 Clause License and +// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +// was not distributed with this source code in the LICENSE file, you can +// obtain it at www.aomedia.org/license/software. If the Alliance for Open +// Media Patent License 1.0 was not distributed with this source code in the +// PATENTS file, you can obtain it at www.aomedia.org/license/patent. + +use std::time::Duration; + +use dora_node_api::arrow::array::UInt8Array; +use dora_node_api::dora_core::config::DataId; +use dora_node_api::{DoraNode, Event, IntoArrow, MetadataParameters, Parameter}; +use eyre::{Context as EyreContext, Result}; +// Encode the same tiny blank frame 30 times +use rav1e::config::RateControlConfig; +use rav1e::config::SpeedSettings; + +use rav1e::*; + +fn bgr_to_yuv(bgr_data: Vec, width: usize, height: usize) -> (Vec, Vec, Vec) { + let mut y_plane = vec![0; width * height]; + let mut u_plane = vec![0; (width / 2) * (height / 2)]; + let mut v_plane = vec![0; (width / 2) * (height / 2)]; + + for y in 0..height { + for x in 0..width { + let bgr_index = (y * width + x) * 3; + let b = bgr_data[bgr_index] as f32; + let g = bgr_data[bgr_index + 1] as f32; + let r = bgr_data[bgr_index + 2] as f32; + + // Corrected YUV conversion formulas + let y_value = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0); + let u_value = (-0.14713 * r - 0.28886 * g + 0.436 * b + 128.0).clamp(0.0, 255.0); + let v_value = (0.615 * r - 0.51499 * g - 0.10001 * b + 128.0).clamp(0.0, 255.0); + + let y_index = y * width + x; + y_plane[y_index] = y_value.round() as u8; + + if x % 2 == 0 && y % 2 == 0 { + let uv_index = (y / 2) * (width / 2) + (x / 2); + u_plane[uv_index] = u_value.round() as u8; + v_plane[uv_index] = v_value.round() as u8; + } + } + } + + (y_plane, u_plane, v_plane) +} + +fn get_yuv_planes(buffer: Vec, width: usize, height: usize) -> (Vec, Vec, Vec) { + // Calculate sizes of Y, U, and V planes for YUV420 format + let y_size = width * height; // Y has full resolution + let uv_width = width / 2; // U and V are subsampled by 2 in both dimensions + let uv_height = height / 2; // U and V are subsampled by 2 in both dimensions + let uv_size = uv_width * uv_height; // Size for U and V planes + + // Ensure the buffer has the correct size + // if buffer.len() != y_size + 2 * uv_size { + // panic!("Invalid buffer size for the given width and height!"); + // } + + // Extract Y, U, and V planes + let y_plane = buffer[0..y_size].to_vec(); + let u_plane = buffer[y_size..y_size + uv_size].to_vec(); + let v_plane = buffer[y_size + uv_size..].to_vec(); + + (y_plane, u_plane, v_plane) +} + +fn main() -> Result<()> { + let mut height = std::env::var("IMAGE_HEIGHT") + .unwrap_or_else(|_| "480".to_string()) + .parse() + .unwrap(); + let mut width = std::env::var("IMAGE_WIDTH") + .unwrap_or_else(|_| "640".to_string()) + .parse() + .unwrap(); + let enc = EncoderConfig { + width, + height, + speed_settings: SpeedSettings::from_preset(8), + low_latency: true, + ..Default::default() + }; + + let cfg = Config::new() + // .with_rate_control(RateControlConfig::new().with_emit_data(true)) + .with_encoder_config(enc.clone()) + .with_threads(16); + + let mut ctx: Context = cfg.new_context().unwrap(); + + let (mut node, mut events) = + DoraNode::init_from_env().context("Could not initialize dora node")?; + + let mut time = std::time::Instant::now(); + loop { + let buffer = match events.recv() { + Some(Event::Input { id, data, metadata }) => { + if let Some(Parameter::Integer(h)) = metadata.parameters.get("height") { + height = *h as usize; + }; + if let Some(Parameter::Integer(w)) = metadata.parameters.get("width") { + width = *w as usize; + }; + let encoding = if let Some(Parameter::String(encoding)) = + metadata.parameters.get("encoding") + { + encoding + } else { + "bgr8" + }; + + if encoding == "bgr8" { + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: Vec = buffer.values().to_vec(); + buffer + // Transpose values from BGR to RGB + // let buffer: Vec = buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect(); + + //un + } else if encoding == "yuv420" { + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: Vec = buffer.values().to_vec(); + buffer + } else if encoding == "rgb8" { + unimplemented!("We haven't worked on additional encodings."); + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: &[u8] = buffer.values(); + let mut f = ctx.new_frame(); + + for p in &mut f.planes { + let stride = (enc.width + p.cfg.xdec) >> p.cfg.xdec; + p.copy_from_raw_u8(&buffer, stride, 1); + } + buffer.to_vec() + } else { + unimplemented!("We haven't worked on additional encodings."); + continue; + } + } + Some(Event::Error(e)) => { + continue; + } + _ => break, + }; + //let (y, u, v) = bgr_to_yuv(buffer, 640 as usize, 480 as usize); + + let (y, u, v) = get_yuv_planes(buffer, width, height); + let mut f = ctx.new_frame(); + + let xdec = f.planes[0].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[0].copy_from_raw_u8(&y, stride, 1); + let xdec = f.planes[1].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[1].copy_from_raw_u8(&u, stride, 1); + let xdec = f.planes[2].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[2].copy_from_raw_u8(&v, stride, 1); + + match ctx.send_frame(f) { + Ok(_) => {} + Err(e) => match e { + EncoderStatus::EnoughData => { + println!("Unable to append frame to the internal queue"); + panic!("Unable to send frame "); + } + _ => { + panic!("Unable to send frame "); + } + }, + } + println!("Frame sent to encoder"); + for _ in 0..1 { + match ctx.receive_packet() { + Ok(pkt) => { + println!("Time to encode: {:?}", time.elapsed()); + time = std::time::Instant::now(); + let data = pkt.data; + println!("frame compression: {:#?}", width * height * 3 / data.len()); + println!("frame size: {:#?}", data.len()); + let arrow = data.into_arrow(); + node.send_output( + DataId::from("frame".to_owned()), + MetadataParameters::default(), + arrow, + ) + .context("could not send output") + .unwrap(); + + break; + } + Err(e) => match e { + EncoderStatus::LimitReached => { + break; + } + EncoderStatus::Encoded => { + break; + } + EncoderStatus::NeedMoreData => { + break; + } + _ => { + panic!("Unable to receive packet",); + } + }, + } + } + } + + Ok(()) +} diff --git a/node-hub/dora-dav1d/Cargo.toml b/node-hub/dora-dav1d/Cargo.toml new file mode 100644 index 00000000..a0940226 --- /dev/null +++ b/node-hub/dora-dav1d/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "dora-dav1d" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +dav1d = "0.10" +bitstream-io = "2.0" +log = "0.4" +structopt = "0.3" +dora-node-api = { workspace = true, features = ["tracing"] } +eyre = "0.6.8" diff --git a/node-hub/dora-dav1d/dataflow.yml b/node-hub/dora-dav1d/dataflow.yml new file mode 100644 index 00000000..b4853804 --- /dev/null +++ b/node-hub/dora-dav1d/dataflow.yml @@ -0,0 +1,53 @@ +nodes: + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: dora/timer/millis/50 + outputs: + - image + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + ENCODING: yuv420 + + - id: av + path: /home/peter/Documents/work/dora/target/release/dora-av1-encoder + build: cargo build -p dora-av1-encoder --release + inputs: + image: camera/image + outputs: + - frame + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: dav1d + path: /home/peter/Documents/work/dora/target/release/dora-dav1d + build: cargo build -p dora-dav1d --release + inputs: + image: av/frame + outputs: + - frame + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: plot + build: pip install -e ../../node-hub/opencv-plot + path: opencv-plot + inputs: + image: dav1d/frame + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: plot2 + build: pip install -e ../../node-hub/opencv-plot + path: opencv-plot + inputs: + image: camera/image + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs new file mode 100644 index 00000000..0dd758b5 --- /dev/null +++ b/node-hub/dora-dav1d/src/main.rs @@ -0,0 +1,189 @@ +use std::time::Duration; + +use dav1d::Settings; +use dora_node_api::{ + arrow::array::UInt8Array, dora_core::config::DataId, DoraNode, Event, IntoArrow, + MetadataParameters, +}; +use eyre::{Context, Result}; + +fn yuv420_to_bgr( + y_plane: &[u8], + u_plane: &[u8], + v_plane: &[u8], + width: usize, + height: usize, +) -> Vec { + let mut rgb_data = vec![0u8; width * height * 3]; // Output RGB data buffer + + for j in 0..height { + for i in 0..width { + let y_idx = j * width + i; // Index in Y plane + let uv_idx = (j / 2) * (width / 2) + (i / 2); // Index in U/V planes + + let y = y_plane[y_idx] as f32; + let u = u_plane[uv_idx] as f32 - 128.0; + let v = v_plane[uv_idx] as f32 - 128.0; + + // Convert YUV to RGB using BT.601 standard formula + let r = (y + 1.402 * v).clamp(0.0, 255.0) as u8; + let g = (y - 0.344136 * u - 0.714136 * v).clamp(0.0, 255.0) as u8; + let b = (y + 1.772 * u).clamp(0.0, 255.0) as u8; + + // Set the RGB values in the output buffer + let rgb_idx = y_idx * 3; + rgb_data[rgb_idx] = b; + rgb_data[rgb_idx + 1] = g; + rgb_data[rgb_idx + 2] = r; + } + } + + rgb_data +} + +fn main() -> Result<()> { + let mut settings = Settings::new(); + settings.set_n_threads(16); + + let height: usize = std::env::var("IMAGE_HEIGHT") + .unwrap_or_else(|_| "480".to_string()) + .parse() + .unwrap(); + let width: usize = std::env::var("IMAGE_WIDTH") + .unwrap_or_else(|_| "640".to_string()) + .parse() + .unwrap(); + + let mut dec = + dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance"); + + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let (mut node, mut events) = + DoraNode::init_from_env().context("Could not initialize dora node")?; + + let _ = std::thread::spawn(move || { + let mut now = std::time::Instant::now(); + loop { + while let Ok(data) = rx.recv_timeout(Duration::from_millis(100)) { + match dec.send_data(data, None, None, None) { + Err(e) if e.is_again() => { + panic!("Error sending data to the decoder: {}", e); + if let Ok(p) = dec.get_picture() { + let mut y = p.plane(dav1d::PlanarImageComponent::Y).to_vec(); + let mut u = p.plane(dav1d::PlanarImageComponent::U).to_vec(); + let mut v = p.plane(dav1d::PlanarImageComponent::V).to_vec(); + y.append(&mut u); + y.append(&mut v); + // let rgb = yuv420_to_rgb(&y, &u, &v, 100, 100); + let arrow = y.into_arrow(); + let mut metadata = MetadataParameters::default(); + metadata.insert( + "width".to_string(), + dora_node_api::Parameter::Integer(640), + ); + metadata.insert( + "height".to_string(), + dora_node_api::Parameter::Integer(480), + ); + metadata.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("yuv420".to_string()), + ); + node.send_output(DataId::from("frame".to_string()), metadata, arrow) + .unwrap(); + println!("Time to decode: {:?}", now.elapsed()); + now = std::time::Instant::now(); + } + // If the decoder did not consume all data, output all + // pending pictures and send pending data to the decoder + // until it is all used up. + //loop { + // handle_pending_pictures(&mut dec, false, &mut node); + + // match dec.send_pending_data() { + // Err(e) if e.is_again() => continue, + // Err(e) => { + // panic!("Error sending pending data to the decoder: {}", e); + // } + // _ => break, + // } + //} + } + Err(e) => { + panic!("Error sending data to the decoder: {}", e); + } + Ok(()) => { + if let Ok(p) = dec.get_picture() { + let mut y = p.plane(dav1d::PlanarImageComponent::Y).to_vec(); + let mut u = p.plane(dav1d::PlanarImageComponent::U).to_vec(); + let mut v = p.plane(dav1d::PlanarImageComponent::V).to_vec(); + // u.iter_mut().for_each(|e| { + // if *e < 128 { + // *e = *e + 128 + // } + // }); + // v.iter_mut().for_each(|e: &mut u8| { + // if *e < 128 { + // *e = *e + 128 + // } + // }); + + // y.append(&mut u); + // y.append(&mut v); + let y = yuv420_to_bgr(&y, &u, &v, width, height); + + let arrow = y.into_arrow(); + let mut metadata = MetadataParameters::default(); + metadata.insert( + "width".to_string(), + dora_node_api::Parameter::Integer( + width.try_into().unwrap_or_default(), + ), + ); + metadata.insert( + "height".to_string(), + dora_node_api::Parameter::Integer( + height.try_into().unwrap_or_default(), + ), + ); + metadata.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("bgr8".to_string()), + ); + node.send_output(DataId::from("frame".to_string()), metadata, arrow) + .unwrap(); + println!("Time to decode: {:?}", now.elapsed()); + println!("Delay: {:#?}", dec.get_frame_delay()); + + now = std::time::Instant::now(); + } + } + } + } + } + }); + loop { + match events.recv() { + Some(Event::Input { + id: _, + data, + metadata: _, + }) => { + let data = data.as_any().downcast_ref::().unwrap(); + let data = data.values().clone(); + + // Send packet to the decoder + + tx.try_send(data).unwrap_or_default(); + // Handle all pending pictures before sending the next data. + // handle_pending_pictures(&mut dec, false, &mut node); + } + None => break, + Some(_) => break, + } + } + // Handle all pending pictures that were not output yet. + // handle_pending_pictures(&mut dec, true, &mut node); + + Ok(()) +} From 26c40c5aacde2ea6401e0c5ef2121d4e431a77f0 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Mar 2025 12:01:54 +0100 Subject: [PATCH 02/26] av1 --- node-hub/dora-av1-encoder/Cargo.toml | 1 + node-hub/dora-av1-encoder/src/main.rs | 128 ++++++++++++-------------- node-hub/dora-dav1d/src/main.rs | 108 ++++++---------------- 3 files changed, 89 insertions(+), 148 deletions(-) diff --git a/node-hub/dora-av1-encoder/Cargo.toml b/node-hub/dora-av1-encoder/Cargo.toml index b0ee3d8a..188ec001 100644 --- a/node-hub/dora-av1-encoder/Cargo.toml +++ b/node-hub/dora-av1-encoder/Cargo.toml @@ -13,3 +13,4 @@ repository.workspace = true rav1e = { version = "0.7.1", features = ["serialize"] } dora-node-api = { workspace = true, features = ["tracing"] } eyre = "0.6.8" +log = "0.4" diff --git a/node-hub/dora-av1-encoder/src/main.rs b/node-hub/dora-av1-encoder/src/main.rs index dbad450e..b53a8684 100644 --- a/node-hub/dora-av1-encoder/src/main.rs +++ b/node-hub/dora-av1-encoder/src/main.rs @@ -13,6 +13,7 @@ use dora_node_api::arrow::array::UInt8Array; use dora_node_api::dora_core::config::DataId; use dora_node_api::{DoraNode, Event, IntoArrow, MetadataParameters, Parameter}; use eyre::{Context as EyreContext, Result}; +use log::warn; // Encode the same tiny blank frame 30 times use rav1e::config::RateControlConfig; use rav1e::config::SpeedSettings; @@ -50,7 +51,7 @@ fn bgr_to_yuv(bgr_data: Vec, width: usize, height: usize) -> (Vec, Vec, width: usize, height: usize) -> (Vec, Vec, Vec) { +fn get_yuv_planes(buffer: &[u8], width: usize, height: usize) -> (&[u8], &[u8], &[u8]) { // Calculate sizes of Y, U, and V planes for YUV420 format let y_size = width * height; // Y has full resolution let uv_width = width / 2; // U and V are subsampled by 2 in both dimensions @@ -63,9 +64,9 @@ fn get_yuv_planes(buffer: Vec, width: usize, height: usize) -> (Vec, Vec // } // Extract Y, U, and V planes - let y_plane = buffer[0..y_size].to_vec(); - let u_plane = buffer[y_size..y_size + uv_size].to_vec(); - let v_plane = buffer[y_size + uv_size..].to_vec(); + let y_plane = &buffer[0..y_size]; //.to_vec(); + let u_plane = &buffer[y_size..y_size + uv_size]; //.to_vec(); + let v_plane = &buffer[y_size + uv_size..]; //.to_vec(); (y_plane, u_plane, v_plane) } @@ -82,7 +83,7 @@ fn main() -> Result<()> { let enc = EncoderConfig { width, height, - speed_settings: SpeedSettings::from_preset(8), + speed_settings: SpeedSettings::from_preset(10), low_latency: true, ..Default::default() }; @@ -91,6 +92,7 @@ fn main() -> Result<()> { // .with_rate_control(RateControlConfig::new().with_emit_data(true)) .with_encoder_config(enc.clone()) .with_threads(16); + cfg.validate()?; let mut ctx: Context = cfg.new_context().unwrap(); @@ -125,8 +127,58 @@ fn main() -> Result<()> { //un } else if encoding == "yuv420" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); - let buffer: Vec = buffer.values().to_vec(); - buffer + let buffer = buffer.values(); //.to_vec(); + + let (y, u, v) = get_yuv_planes(buffer, width, height); + let mut f = ctx.new_frame(); + + let xdec = f.planes[0].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[0].copy_from_raw_u8(&y, stride, 1); + let xdec = f.planes[1].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[1].copy_from_raw_u8(&u, stride, 1); + let xdec = f.planes[2].cfg.xdec; + let stride = (enc.width + xdec) >> xdec; + f.planes[2].copy_from_raw_u8(&v, stride, 1); + + match ctx.send_frame(f) { + Ok(_) => {} + Err(e) => match e { + EncoderStatus::EnoughData => { + warn!("Unable to send frame "); + } + _ => { + warn!("Unable to send frame "); + } + }, + } + match ctx.receive_packet() { + Ok(pkt) => { + println!("Time to encode: {:?}", time.elapsed()); + time = std::time::Instant::now(); + let data = pkt.data; + println!("frame compression: {:#?}", width * height * 3 / data.len()); + println!("frame size: {:#?}", data.len()); + let arrow = data.into_arrow(); + node.send_output( + DataId::from("frame".to_owned()), + MetadataParameters::default(), + arrow, + ) + .context("could not send output") + .unwrap(); + } + Err(e) => match e { + EncoderStatus::LimitReached => {} + EncoderStatus::Encoded => {} + EncoderStatus::NeedMoreData => {} + _ => { + panic!("Unable to receive packet",); + } + }, + } + vec![] } else if encoding == "rgb8" { unimplemented!("We haven't worked on additional encodings."); let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); @@ -149,68 +201,6 @@ fn main() -> Result<()> { _ => break, }; //let (y, u, v) = bgr_to_yuv(buffer, 640 as usize, 480 as usize); - - let (y, u, v) = get_yuv_planes(buffer, width, height); - let mut f = ctx.new_frame(); - - let xdec = f.planes[0].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; - f.planes[0].copy_from_raw_u8(&y, stride, 1); - let xdec = f.planes[1].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; - f.planes[1].copy_from_raw_u8(&u, stride, 1); - let xdec = f.planes[2].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; - f.planes[2].copy_from_raw_u8(&v, stride, 1); - - match ctx.send_frame(f) { - Ok(_) => {} - Err(e) => match e { - EncoderStatus::EnoughData => { - println!("Unable to append frame to the internal queue"); - panic!("Unable to send frame "); - } - _ => { - panic!("Unable to send frame "); - } - }, - } - println!("Frame sent to encoder"); - for _ in 0..1 { - match ctx.receive_packet() { - Ok(pkt) => { - println!("Time to encode: {:?}", time.elapsed()); - time = std::time::Instant::now(); - let data = pkt.data; - println!("frame compression: {:#?}", width * height * 3 / data.len()); - println!("frame size: {:#?}", data.len()); - let arrow = data.into_arrow(); - node.send_output( - DataId::from("frame".to_owned()), - MetadataParameters::default(), - arrow, - ) - .context("could not send output") - .unwrap(); - - break; - } - Err(e) => match e { - EncoderStatus::LimitReached => { - break; - } - EncoderStatus::Encoded => { - break; - } - EncoderStatus::NeedMoreData => { - break; - } - _ => { - panic!("Unable to receive packet",); - } - }, - } - } } Ok(()) diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index 0dd758b5..22c160da 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -6,6 +6,7 @@ use dora_node_api::{ MetadataParameters, }; use eyre::{Context, Result}; +use log::warn; fn yuv420_to_bgr( y_plane: &[u8], @@ -44,6 +45,7 @@ fn yuv420_to_bgr( fn main() -> Result<()> { let mut settings = Settings::new(); settings.set_n_threads(16); + settings.set_max_frame_delay(1); let height: usize = std::env::var("IMAGE_HEIGHT") .unwrap_or_else(|_| "480".to_string()) @@ -57,76 +59,41 @@ fn main() -> Result<()> { let mut dec = dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance"); - let (tx, rx) = std::sync::mpsc::sync_channel(1); let (mut node, mut events) = DoraNode::init_from_env().context("Could not initialize dora node")?; - let _ = std::thread::spawn(move || { - let mut now = std::time::Instant::now(); - loop { - while let Ok(data) = rx.recv_timeout(Duration::from_millis(100)) { + let mut now = std::time::Instant::now(); + loop { + match events.recv() { + Some(Event::Input { + id: _, + data, + metadata: _, + }) => { + let data = data.as_any().downcast_ref::().unwrap(); + let data = data.values().clone(); + + // Send packet to the decoder + match dec.send_data(data, None, None, None) { - Err(e) if e.is_again() => { - panic!("Error sending data to the decoder: {}", e); - if let Ok(p) = dec.get_picture() { - let mut y = p.plane(dav1d::PlanarImageComponent::Y).to_vec(); - let mut u = p.plane(dav1d::PlanarImageComponent::U).to_vec(); - let mut v = p.plane(dav1d::PlanarImageComponent::V).to_vec(); - y.append(&mut u); - y.append(&mut v); - // let rgb = yuv420_to_rgb(&y, &u, &v, 100, 100); - let arrow = y.into_arrow(); - let mut metadata = MetadataParameters::default(); - metadata.insert( - "width".to_string(), - dora_node_api::Parameter::Integer(640), - ); - metadata.insert( - "height".to_string(), - dora_node_api::Parameter::Integer(480), - ); - metadata.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("yuv420".to_string()), - ); - node.send_output(DataId::from("frame".to_string()), metadata, arrow) - .unwrap(); - println!("Time to decode: {:?}", now.elapsed()); - now = std::time::Instant::now(); - } - // If the decoder did not consume all data, output all - // pending pictures and send pending data to the decoder - // until it is all used up. - //loop { - // handle_pending_pictures(&mut dec, false, &mut node); - - // match dec.send_pending_data() { - // Err(e) if e.is_again() => continue, - // Err(e) => { - // panic!("Error sending pending data to the decoder: {}", e); - // } - // _ => break, - // } - //} - } Err(e) => { - panic!("Error sending data to the decoder: {}", e); + warn!("Error sending data to the decoder: {}", e); } Ok(()) => { if let Ok(p) = dec.get_picture() { - let mut y = p.plane(dav1d::PlanarImageComponent::Y).to_vec(); - let mut u = p.plane(dav1d::PlanarImageComponent::U).to_vec(); - let mut v = p.plane(dav1d::PlanarImageComponent::V).to_vec(); - // u.iter_mut().for_each(|e| { - // if *e < 128 { - // *e = *e + 128 - // } - // }); - // v.iter_mut().for_each(|e: &mut u8| { - // if *e < 128 { - // *e = *e + 128 - // } - // }); + let mut y = p.plane(dav1d::PlanarImageComponent::Y); //.to_vec(); + let mut u = p.plane(dav1d::PlanarImageComponent::U); //.to_vec(); + let mut v = p.plane(dav1d::PlanarImageComponent::V); //.to_vec(); + // u.iter_mut().for_each(|e| { + // if *e < 128 { + // *e = *e + 128 + // } + // }); + // v.iter_mut().for_each(|e: &mut u8| { + // if *e < 128 { + // *e = *e + 128 + // } + // }); // y.append(&mut u); // y.append(&mut v); @@ -153,28 +120,11 @@ fn main() -> Result<()> { node.send_output(DataId::from("frame".to_string()), metadata, arrow) .unwrap(); println!("Time to decode: {:?}", now.elapsed()); - println!("Delay: {:#?}", dec.get_frame_delay()); now = std::time::Instant::now(); } } } - } - } - }); - loop { - match events.recv() { - Some(Event::Input { - id: _, - data, - metadata: _, - }) => { - let data = data.as_any().downcast_ref::().unwrap(); - let data = data.values().clone(); - - // Send packet to the decoder - - tx.try_send(data).unwrap_or_default(); // Handle all pending pictures before sending the next data. // handle_pending_pictures(&mut dec, false, &mut node); } From f1bc8a7b1eb1465d83a192f0968cca55412eb08a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Mar 2025 14:02:23 +0100 Subject: [PATCH 03/26] Minor improvement --- node-hub/dora-av1-encoder/src/main.rs | 24 ++++++++---------------- node-hub/dora-dav1d/src/main.rs | 4 +++- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/node-hub/dora-av1-encoder/src/main.rs b/node-hub/dora-av1-encoder/src/main.rs index b53a8684..75218ec0 100644 --- a/node-hub/dora-av1-encoder/src/main.rs +++ b/node-hub/dora-av1-encoder/src/main.rs @@ -7,15 +7,12 @@ // Media Patent License 1.0 was not distributed with this source code in the // PATENTS file, you can obtain it at www.aomedia.org/license/patent. -use std::time::Duration; - use dora_node_api::arrow::array::UInt8Array; use dora_node_api::dora_core::config::DataId; use dora_node_api::{DoraNode, Event, IntoArrow, MetadataParameters, Parameter}; use eyre::{Context as EyreContext, Result}; use log::warn; // Encode the same tiny blank frame 30 times -use rav1e::config::RateControlConfig; use rav1e::config::SpeedSettings; use rav1e::*; @@ -90,18 +87,14 @@ fn main() -> Result<()> { let cfg = Config::new() // .with_rate_control(RateControlConfig::new().with_emit_data(true)) - .with_encoder_config(enc.clone()) - .with_threads(16); + .with_encoder_config(enc.clone()); //.with_threads(16); cfg.validate()?; - let mut ctx: Context = cfg.new_context().unwrap(); - let (mut node, mut events) = DoraNode::init_from_env().context("Could not initialize dora node")?; - let mut time = std::time::Instant::now(); loop { - let buffer = match events.recv() { + let _buffer = match events.recv() { Some(Event::Input { id, data, metadata }) => { if let Some(Parameter::Integer(h)) = metadata.parameters.get("height") { height = *h as usize; @@ -130,6 +123,7 @@ fn main() -> Result<()> { let buffer = buffer.values(); //.to_vec(); let (y, u, v) = get_yuv_planes(buffer, width, height); + let mut ctx: Context = cfg.new_context().unwrap(); let mut f = ctx.new_frame(); let xdec = f.planes[0].cfg.xdec; @@ -153,13 +147,12 @@ fn main() -> Result<()> { } }, } + ctx.flush(); match ctx.receive_packet() { Ok(pkt) => { - println!("Time to encode: {:?}", time.elapsed()); - time = std::time::Instant::now(); let data = pkt.data; - println!("frame compression: {:#?}", width * height * 3 / data.len()); - println!("frame size: {:#?}", data.len()); + println!("Packet data: {:?}", data.len()); + println!("Compression: {:?}", height * width * 3 / data.len()); let arrow = data.into_arrow(); node.send_output( DataId::from("frame".to_owned()), @@ -183,6 +176,7 @@ fn main() -> Result<()> { unimplemented!("We haven't worked on additional encodings."); let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); let buffer: &[u8] = buffer.values(); + let mut ctx: Context = cfg.new_context().unwrap(); let mut f = ctx.new_frame(); for p in &mut f.planes { @@ -192,15 +186,13 @@ fn main() -> Result<()> { buffer.to_vec() } else { unimplemented!("We haven't worked on additional encodings."); - continue; } } - Some(Event::Error(e)) => { + Some(Event::Error(_e)) => { continue; } _ => break, }; - //let (y, u, v) = bgr_to_yuv(buffer, 640 as usize, 480 as usize); } Ok(()) diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index 22c160da..1486601e 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -64,6 +64,7 @@ fn main() -> Result<()> { let mut now = std::time::Instant::now(); loop { + let time = std::time::Instant::now(); match events.recv() { Some(Event::Input { id: _, @@ -81,6 +82,7 @@ fn main() -> Result<()> { } Ok(()) => { if let Ok(p) = dec.get_picture() { + // println!("Time to decode: {:?}", time.elapsed()); let mut y = p.plane(dav1d::PlanarImageComponent::Y); //.to_vec(); let mut u = p.plane(dav1d::PlanarImageComponent::U); //.to_vec(); let mut v = p.plane(dav1d::PlanarImageComponent::V); //.to_vec(); @@ -119,7 +121,7 @@ fn main() -> Result<()> { ); node.send_output(DataId::from("frame".to_string()), metadata, arrow) .unwrap(); - println!("Time to decode: {:?}", now.elapsed()); + //println!("Time to decode: {:?}", now.elapsed()); now = std::time::Instant::now(); } From fff5597e9aef8d3fde6ad61826061cba181f5dbc Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Mar 2025 19:20:27 +0100 Subject: [PATCH 04/26] minor av1 typo --- node-hub/dora-av1-encoder/src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/node-hub/dora-av1-encoder/src/main.rs b/node-hub/dora-av1-encoder/src/main.rs index 75218ec0..0a177ad7 100644 --- a/node-hub/dora-av1-encoder/src/main.rs +++ b/node-hub/dora-av1-encoder/src/main.rs @@ -151,8 +151,6 @@ fn main() -> Result<()> { match ctx.receive_packet() { Ok(pkt) => { let data = pkt.data; - println!("Packet data: {:?}", data.len()); - println!("Compression: {:?}", height * width * 3 / data.len()); let arrow = data.into_arrow(); node.send_output( DataId::from("frame".to_owned()), From b1ab8abaf5753ab25d59ae7fc78db4b2bbff3e49 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 28 Mar 2025 18:43:06 +0100 Subject: [PATCH 05/26] Enabling remote av1 communication --- Cargo.lock | 138 +++++++++++++++--- Cargo.toml | 2 +- binaries/daemon/src/lib.rs | 22 ++- node-hub/dora-av1-encoder/example.py | 22 --- node-hub/dora-dav1d/dataflow.yml | 84 +++++++++-- node-hub/dora-dav1d/src/main.rs | 70 ++------- .../Cargo.toml | 2 +- .../src/main.rs | 49 ++++--- 8 files changed, 248 insertions(+), 141 deletions(-) delete mode 100644 node-hub/dora-av1-encoder/example.py rename node-hub/{dora-av1-encoder => dora-rav1e}/Cargo.toml (94%) rename node-hub/{dora-av1-encoder => dora-rav1e}/src/main.rs (84%) diff --git a/Cargo.lock b/Cargo.lock index 9482a998..40d2d356 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,6 +383,17 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "array-init" version = "2.1.0" @@ -406,6 +417,9 @@ name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +dependencies = [ + "serde", +] [[package]] name = "arrow" @@ -1317,7 +1331,7 @@ dependencies = [ "lab", "num-traits", "rayon", - "thiserror 1.0.66", + "thiserror 1.0.69", "v_frame", ] @@ -1330,7 +1344,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 7.1.3", "num-rational", "serde", "v_frame", @@ -1555,6 +1569,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "block" version = "0.1.6" @@ -2055,7 +2075,7 @@ dependencies = [ "bitflags 1.3.2", "strsim 0.8.0", "textwrap 0.11.0", - "unicode-width", + "unicode-width 0.1.14", "vec_map", ] @@ -2073,7 +2093,7 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.1", + "textwrap 0.16.2", ] [[package]] @@ -2105,7 +2125,7 @@ version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06f5378ea264ad4f82bbc826628b5aad714a75abf6ece087e923010eb937fb6" dependencies = [ - "clap 4.5.20", + "clap 4.5.32", ] [[package]] @@ -2757,6 +2777,28 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" +[[package]] +name = "dav1d" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4b54a40baf633a71c6f0fb49494a7e4ee7bc26f3e727212b6cb915aa1ea1e1" +dependencies = [ + "av-data", + "bitflags 2.9.0", + "dav1d-sys", + "static_assertions", +] + +[[package]] +name = "dav1d-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ecb1c5e8f4dc438eedc1b534a54672fb0e0a56035dae6b50162787bd2c50e95" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "defmac" version = "0.1.3" @@ -3019,15 +3061,6 @@ dependencies = [ "num", ] -[[package]] -name = "dora-av1-encoder" -version = "0.3.10" -dependencies = [ - "dora-node-api", - "eyre", - "rav1e", -] - [[package]] name = "dora-cli" version = "0.3.10" @@ -3399,6 +3432,16 @@ dependencies = [ "safer-ffi", ] +[[package]] +name = "dora-rav1e" +version = "0.3.10" +dependencies = [ + "dora-node-api", + "eyre", + "log", + "rav1e", +] + [[package]] name = "dora-record" version = "0.3.10" @@ -5828,6 +5871,17 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "interprocess" version = "2.2.3" @@ -5954,6 +6008,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -5978,6 +6041,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "ivf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552f657140ee72c552b728601179c10abea14cd7d815de2d75d75dea42485eca" +dependencies = [ + "bitstream-io", +] + [[package]] name = "jiff" version = "0.2.4" @@ -7283,6 +7355,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "notify" version = "5.2.0" @@ -9260,7 +9338,7 @@ dependencies = [ "built", "cc", "cfg-if 1.0.0", - "clap 4.5.20", + "clap 4.5.32", "clap_complete", "console", "fern", @@ -9273,22 +9351,22 @@ dependencies = [ "maybe-rayon", "nasm-rs 0.2.5", "new_debug_unreachable", - "nom", + "nom 7.1.3", "noop_proc_macro", "num-derive", "num-traits", "once_cell", "paste", "profiling", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "scan_fmt", "serde", "serde-big-array", "signal-hook", "simd_helpers", "system-deps", - "thiserror 1.0.66", + "thiserror 1.0.69", "toml", "v_frame", "y4m", @@ -13028,6 +13106,19 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml", + "version-compare", +] + [[package]] name = "tabwriter" version = "1.4.1" @@ -13118,6 +13209,15 @@ dependencies = [ "libc", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "textwrap" version = "0.16.2" diff --git a/Cargo.toml b/Cargo.toml index 50bbee92..d7a82e78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ members = [ "node-hub/dora-kit-car", "node-hub/dora-object-to-pose", "node-hub/dora-mistral-rs", - "node-hub/dora-av1-encoder", + "node-hub/dora-rav1e", "node-hub/dora-dav1d", "libraries/extensions/ros2-bridge", "libraries/extensions/ros2-bridge/msg-gen", diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index de531fd0..2e08290f 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -251,11 +251,29 @@ impl Daemon { None => None, }; - let zenoh_config = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) { + let mut zenoh_config = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) { Ok(path) => zenoh::Config::from_file(&path) .map_err(|e| eyre!(e)) .wrap_err_with(|| format!("failed to read zenoh config from {path}"))?, - Err(std::env::VarError::NotPresent) => zenoh::Config::default(), + Err(std::env::VarError::NotPresent) => { + let mut zenoh_config = zenoh::Config::default(); + + if let Some(addr) = coordinator_addr { + zenoh_config + .insert_json5( + "connect/endpoints", + &format!(r#"["tcp/{}:5456"]"#, addr.ip()), + ) + .unwrap(); + zenoh_config + .insert_json5( + "listen/endpoints", + r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:0", "tcp/[::]:5456"] }"#, + ) + .unwrap(); + }; + zenoh_config + } Err(std::env::VarError::NotUnicode(_)) => eyre::bail!( "{} env variable is not valid unicode", zenoh::Config::DEFAULT_CONFIG_PATH_ENV diff --git a/node-hub/dora-av1-encoder/example.py b/node-hub/dora-av1-encoder/example.py deleted file mode 100644 index 80d54b89..00000000 --- a/node-hub/dora-av1-encoder/example.py +++ /dev/null @@ -1,22 +0,0 @@ -import av -import cv2 -import numpy as np - -# Open the AV1 video file -container = av.open("video.av1") - -# Get the video stream -stream = next(s for s in container.streams if s.type == "video") - -# Iterate over the video frames -for frame in container.decode(stream): - # Convert the frame to a NumPy array - img = frame.to_ndarray(format="bgr24") - - # Display the frame - cv2.imshow("Frame", img) - if cv2.waitKey(1) & 0xFF == ord("q"): - break - -# Release the resources -cv2.destroyAllWindows() diff --git a/node-hub/dora-dav1d/dataflow.yml b/node-hub/dora-dav1d/dataflow.yml index b4853804..ee76b1f3 100644 --- a/node-hub/dora-dav1d/dataflow.yml +++ b/node-hub/dora-dav1d/dataflow.yml @@ -2,6 +2,8 @@ nodes: - id: camera build: pip install ../../node-hub/opencv-video-capture path: opencv-video-capture + _unstable_deploy: + machine: encoder inputs: tick: dora/timer/millis/50 outputs: @@ -11,43 +13,95 @@ nodes: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 ENCODING: yuv420 + - id: camera2 + path: dora-pyrealsense + _unstable_deploy: + machine: encoder + inputs: + tick: dora/timer/millis/10 + outputs: + - image + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + ENCODING: jpeg - - id: av - path: /home/peter/Documents/work/dora/target/release/dora-av1-encoder - build: cargo build -p dora-av1-encoder --release + - id: echo + build: pip install -e ../../node-hub/dora-echo + path: dora-echo + _unstable_deploy: + machine: decoder inputs: - image: camera/image + image: camera2/image + outputs: + - image + + - id: rav1e-local + path: dora-av1-encoder + build: cargo build -p dora-av1-encoder --release + _unstable_deploy: + machine: encoder + #inputs: + # image: camera/image outputs: - frame env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - - id: dav1d - path: /home/peter/Documents/work/dora/target/release/dora-dav1d + - id: dav1d-remote + path: dora-dav1d build: cargo build -p dora-dav1d --release + _unstable_deploy: + machine: decoder inputs: - image: av/frame + image: av-local/frame outputs: - frame env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - - id: plot - build: pip install -e ../../node-hub/opencv-plot - path: opencv-plot + - id: rav1e-remote + path: dora-av1-encoder + build: cargo build -p dora-av1-encoder --release + _unstable_deploy: + machine: decoder inputs: - image: dav1d/frame + image: dav1d-remote/image + outputs: + - frame env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - - id: plot2 - build: pip install -e ../../node-hub/opencv-plot - path: opencv-plot + - id: dav1d-local + path: dora-dav1d + build: cargo build -p dora-dav1d --release + _unstable_deploy: + machine: decoder inputs: - image: camera/image + image: rav1e-local/frame + outputs: + - frame env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 + + - id: plot + build: pip install -e ../../node-hub/dora-rerun + _unstable_deploy: + machine: encoder + path: dora-rerun + inputs: + image: dav1d-local/frame + # image: echo/image + #image_2: camera6/image + + - id: plot2 + build: pip install -e ../../node-hub/opencv-plot + _unstable_deploy: + machine: encoder + path: opencv-plot + inputs: + image: echo/image diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index 1486601e..f337dd41 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -1,10 +1,5 @@ -use std::time::Duration; - use dav1d::Settings; -use dora_node_api::{ - arrow::array::UInt8Array, dora_core::config::DataId, DoraNode, Event, IntoArrow, - MetadataParameters, -}; +use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow}; use eyre::{Context, Result}; use log::warn; @@ -44,38 +39,25 @@ fn yuv420_to_bgr( fn main() -> Result<()> { let mut settings = Settings::new(); - settings.set_n_threads(16); + // settings.set_n_threads(16); settings.set_max_frame_delay(1); - let height: usize = std::env::var("IMAGE_HEIGHT") - .unwrap_or_else(|_| "480".to_string()) - .parse() - .unwrap(); - let width: usize = std::env::var("IMAGE_WIDTH") - .unwrap_or_else(|_| "640".to_string()) - .parse() - .unwrap(); - let mut dec = dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance"); let (mut node, mut events) = DoraNode::init_from_env().context("Could not initialize dora node")?; - let mut now = std::time::Instant::now(); loop { - let time = std::time::Instant::now(); match events.recv() { Some(Event::Input { - id: _, + id, data, - metadata: _, + mut metadata, }) => { let data = data.as_any().downcast_ref::().unwrap(); let data = data.values().clone(); - // Send packet to the decoder - match dec.send_data(data, None, None, None) { Err(e) => { warn!("Error sending data to the decoder: {}", e); @@ -83,59 +65,25 @@ fn main() -> Result<()> { Ok(()) => { if let Ok(p) = dec.get_picture() { // println!("Time to decode: {:?}", time.elapsed()); - let mut y = p.plane(dav1d::PlanarImageComponent::Y); //.to_vec(); - let mut u = p.plane(dav1d::PlanarImageComponent::U); //.to_vec(); - let mut v = p.plane(dav1d::PlanarImageComponent::V); //.to_vec(); - // u.iter_mut().for_each(|e| { - // if *e < 128 { - // *e = *e + 128 - // } - // }); - // v.iter_mut().for_each(|e: &mut u8| { - // if *e < 128 { - // *e = *e + 128 - // } - // }); - - // y.append(&mut u); - // y.append(&mut v); + let y = p.plane(dav1d::PlanarImageComponent::Y); + let u = p.plane(dav1d::PlanarImageComponent::U); + let v = p.plane(dav1d::PlanarImageComponent::V); let y = yuv420_to_bgr(&y, &u, &v, width, height); let arrow = y.into_arrow(); - let mut metadata = MetadataParameters::default(); - metadata.insert( - "width".to_string(), - dora_node_api::Parameter::Integer( - width.try_into().unwrap_or_default(), - ), - ); - metadata.insert( - "height".to_string(), - dora_node_api::Parameter::Integer( - height.try_into().unwrap_or_default(), - ), - ); - metadata.insert( + metadata.parameters.insert( "encoding".to_string(), dora_node_api::Parameter::String("bgr8".to_string()), ); - node.send_output(DataId::from("frame".to_string()), metadata, arrow) - .unwrap(); - //println!("Time to decode: {:?}", now.elapsed()); - - now = std::time::Instant::now(); + node.send_output(id, metadata.parameters, arrow).unwrap(); } } } - // Handle all pending pictures before sending the next data. - // handle_pending_pictures(&mut dec, false, &mut node); } None => break, Some(_) => break, } } - // Handle all pending pictures that were not output yet. - // handle_pending_pictures(&mut dec, true, &mut node); Ok(()) } diff --git a/node-hub/dora-av1-encoder/Cargo.toml b/node-hub/dora-rav1e/Cargo.toml similarity index 94% rename from node-hub/dora-av1-encoder/Cargo.toml rename to node-hub/dora-rav1e/Cargo.toml index 188ec001..10a6bcda 100644 --- a/node-hub/dora-av1-encoder/Cargo.toml +++ b/node-hub/dora-rav1e/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "dora-av1-encoder" +name = "dora-rav1e" edition = "2021" version.workspace = true description.workspace = true diff --git a/node-hub/dora-av1-encoder/src/main.rs b/node-hub/dora-rav1e/src/main.rs similarity index 84% rename from node-hub/dora-av1-encoder/src/main.rs rename to node-hub/dora-rav1e/src/main.rs index 0a177ad7..2931f3b3 100644 --- a/node-hub/dora-av1-encoder/src/main.rs +++ b/node-hub/dora-rav1e/src/main.rs @@ -7,9 +7,10 @@ // Media Patent License 1.0 was not distributed with this source code in the // PATENTS file, you can obtain it at www.aomedia.org/license/patent. +use std::env::var; + use dora_node_api::arrow::array::UInt8Array; -use dora_node_api::dora_core::config::DataId; -use dora_node_api::{DoraNode, Event, IntoArrow, MetadataParameters, Parameter}; +use dora_node_api::{DoraNode, Event, IntoArrow, Parameter}; use eyre::{Context as EyreContext, Result}; use log::warn; // Encode the same tiny blank frame 30 times @@ -77,17 +78,15 @@ fn main() -> Result<()> { .unwrap_or_else(|_| "640".to_string()) .parse() .unwrap(); - let enc = EncoderConfig { + let speed = var("RAV1E_SPEED").map(|s| s.parse().unwrap()).unwrap_or(10); + let mut enc = EncoderConfig { width, height, - speed_settings: SpeedSettings::from_preset(10), + speed_settings: SpeedSettings::from_preset(speed), low_latency: true, ..Default::default() }; - - let cfg = Config::new() - // .with_rate_control(RateControlConfig::new().with_emit_data(true)) - .with_encoder_config(enc.clone()); //.with_threads(16); + let cfg = Config::new().with_encoder_config(enc.clone()); cfg.validate()?; let (mut node, mut events) = @@ -95,7 +94,11 @@ fn main() -> Result<()> { loop { let _buffer = match events.recv() { - Some(Event::Input { id, data, metadata }) => { + Some(Event::Input { + id, + data, + mut metadata, + }) => { if let Some(Parameter::Integer(h)) = metadata.parameters.get("height") { height = *h as usize; }; @@ -109,7 +112,14 @@ fn main() -> Result<()> { } else { "bgr8" }; - + enc = EncoderConfig { + width, + height, + speed_settings: SpeedSettings::from_preset(speed), + low_latency: true, + ..Default::default() + }; + let cfg = Config::new().with_encoder_config(enc.clone()); if encoding == "bgr8" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); let buffer: Vec = buffer.values().to_vec(); @@ -127,13 +137,13 @@ fn main() -> Result<()> { let mut f = ctx.new_frame(); let xdec = f.planes[0].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; + let stride = (width + xdec) >> xdec; f.planes[0].copy_from_raw_u8(&y, stride, 1); let xdec = f.planes[1].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; + let stride = (width + xdec) >> xdec; f.planes[1].copy_from_raw_u8(&u, stride, 1); let xdec = f.planes[2].cfg.xdec; - let stride = (enc.width + xdec) >> xdec; + let stride = (width + xdec) >> xdec; f.planes[2].copy_from_raw_u8(&v, stride, 1); match ctx.send_frame(f) { @@ -147,18 +157,17 @@ fn main() -> Result<()> { } }, } + metadata + .parameters + .insert("encoding".to_string(), Parameter::String("av1".to_string())); ctx.flush(); match ctx.receive_packet() { Ok(pkt) => { let data = pkt.data; let arrow = data.into_arrow(); - node.send_output( - DataId::from("frame".to_owned()), - MetadataParameters::default(), - arrow, - ) - .context("could not send output") - .unwrap(); + node.send_output(id, metadata.parameters, arrow) + .context("could not send output") + .unwrap(); } Err(e) => match e { EncoderStatus::LimitReached => {} From b4c225f5268e8214bb2dcf2e0246ea7ed8e9c89d Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sat, 29 Mar 2025 17:36:32 +0100 Subject: [PATCH 06/26] Adding support for bgr8 in rav1e --- node-hub/dora-dav1d/src/main.rs | 17 +++++++++-- node-hub/dora-rav1e/src/main.rs | 52 +++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index f337dd41..0dbe2106 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -1,5 +1,5 @@ use dav1d::Settings; -use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow}; +use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow, Parameter}; use eyre::{Context, Result}; use log::warn; @@ -64,7 +64,20 @@ fn main() -> Result<()> { } Ok(()) => { if let Ok(p) = dec.get_picture() { - // println!("Time to decode: {:?}", time.elapsed()); + let height = if let Some(Parameter::Integer(h)) = + metadata.parameters.get("height") + { + *h as usize + } else { + 640 + }; + let width = if let Some(Parameter::Integer(w)) = + metadata.parameters.get("width") + { + *w as usize + } else { + 480 + }; let y = p.plane(dav1d::PlanarImageComponent::Y); let u = p.plane(dav1d::PlanarImageComponent::U); let v = p.plane(dav1d::PlanarImageComponent::V); diff --git a/node-hub/dora-rav1e/src/main.rs b/node-hub/dora-rav1e/src/main.rs index 2931f3b3..bff6cf33 100644 --- a/node-hub/dora-rav1e/src/main.rs +++ b/node-hub/dora-rav1e/src/main.rs @@ -18,7 +18,7 @@ use rav1e::config::SpeedSettings; use rav1e::*; -fn bgr_to_yuv(bgr_data: Vec, width: usize, height: usize) -> (Vec, Vec, Vec) { +fn bgr8_to_yuv420(bgr_data: Vec, width: usize, height: usize) -> (Vec, Vec, Vec) { let mut y_plane = vec![0; width * height]; let mut u_plane = vec![0; (width / 2) * (height / 2)]; let mut v_plane = vec![0; (width / 2) * (height / 2)]; @@ -123,11 +123,57 @@ fn main() -> Result<()> { if encoding == "bgr8" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); let buffer: Vec = buffer.values().to_vec(); - buffer + let (y, u, v) = bgr8_to_yuv420(buffer, width, height); + // Transpose values from BGR to RGB // let buffer: Vec = buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect(); - //un + let mut ctx: Context = cfg.new_context().unwrap(); + let mut f = ctx.new_frame(); + + let xdec = f.planes[0].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[0].copy_from_raw_u8(&y, stride, 1); + let xdec = f.planes[1].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[1].copy_from_raw_u8(&u, stride, 1); + let xdec = f.planes[2].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[2].copy_from_raw_u8(&v, stride, 1); + + match ctx.send_frame(f) { + Ok(_) => {} + Err(e) => match e { + EncoderStatus::EnoughData => { + warn!("Unable to send frame "); + } + _ => { + warn!("Unable to send frame "); + } + }, + } + metadata + .parameters + .insert("encoding".to_string(), Parameter::String("av1".to_string())); + ctx.flush(); + match ctx.receive_packet() { + Ok(pkt) => { + let data = pkt.data; + let arrow = data.into_arrow(); + node.send_output(id, metadata.parameters, arrow) + .context("could not send output") + .unwrap(); + } + Err(e) => match e { + EncoderStatus::LimitReached => {} + EncoderStatus::Encoded => {} + EncoderStatus::NeedMoreData => {} + _ => { + panic!("Unable to receive packet",); + } + }, + } + vec![] } else if encoding == "yuv420" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); let buffer = buffer.values(); //.to_vec(); From 73b9823dba243d1f488ecb2de988d53fc6bd2aa7 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Wed, 2 Apr 2025 15:20:28 +0200 Subject: [PATCH 07/26] Enabling mono16 depth frame --- binaries/daemon/src/lib.rs | 48 ++++++++++++++++-------- node-hub/dora-dav1d/Cargo.toml | 1 + node-hub/dora-dav1d/dataflow.yml | 63 +++++++++++++------------------- node-hub/dora-dav1d/src/main.rs | 40 ++++++++++++++------ node-hub/dora-rav1e/src/main.rs | 52 +++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 66 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 2e08290f..474e5a29 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -251,10 +251,16 @@ impl Daemon { None => None, }; - let mut zenoh_config = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) { - Ok(path) => zenoh::Config::from_file(&path) - .map_err(|e| eyre!(e)) - .wrap_err_with(|| format!("failed to read zenoh config from {path}"))?, + let zenoh_session = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) { + Ok(path) => { + let zenoh_config = zenoh::Config::from_file(&path) + .map_err(|e| eyre!(e)) + .wrap_err_with(|| format!("failed to read zenoh config from {path}"))?; + zenoh::open(zenoh_config) + .await + .map_err(|e| eyre!(e)) + .context("failed to open zenoh session")? + } Err(std::env::VarError::NotPresent) => { let mut zenoh_config = zenoh::Config::default(); @@ -266,24 +272,36 @@ impl Daemon { ) .unwrap(); zenoh_config - .insert_json5( - "listen/endpoints", - r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:0", "tcp/[::]:5456"] }"#, - ) - .unwrap(); + .insert_json5( + "listen/endpoints", + r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#, + ) + .unwrap(); + zenoh_config + .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) + .unwrap(); + if cfg!(not(target_os = "linux")) { + warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); + zenoh_config + .insert_json5("scouting/multicast", r#"{ enabled: false }"#) + .unwrap(); + } }; - zenoh_config + if let Ok(zenoh_session) = zenoh::open(zenoh_config).await { + zenoh_session + } else { + warn!("failed to open zenoh session, retrying with default config"); + zenoh::open(zenoh::Config::default()) + .await + .map_err(|e| eyre!(e)) + .wrap_err("failed to open zenoh session")? + } } Err(std::env::VarError::NotUnicode(_)) => eyre::bail!( "{} env variable is not valid unicode", zenoh::Config::DEFAULT_CONFIG_PATH_ENV ), }; - let zenoh_session = zenoh::open(zenoh_config) - .await - .map_err(|e| eyre!(e)) - .context("failed to open zenoh session")?; - let (dora_events_tx, dora_events_rx) = mpsc::channel(5); let daemon = Self { logger: Logger { diff --git a/node-hub/dora-dav1d/Cargo.toml b/node-hub/dora-dav1d/Cargo.toml index a0940226..af76e7e6 100644 --- a/node-hub/dora-dav1d/Cargo.toml +++ b/node-hub/dora-dav1d/Cargo.toml @@ -11,3 +11,4 @@ log = "0.4" structopt = "0.3" dora-node-api = { workspace = true, features = ["tracing"] } eyre = "0.6.8" +bytemuck = "1.7.0" diff --git a/node-hub/dora-dav1d/dataflow.yml b/node-hub/dora-dav1d/dataflow.yml index ee76b1f3..068ecd76 100644 --- a/node-hub/dora-dav1d/dataflow.yml +++ b/node-hub/dora-dav1d/dataflow.yml @@ -9,42 +9,29 @@ nodes: outputs: - image env: - CAPTURE_PATH: 0 + CAPTURE_PATH: 1 IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - ENCODING: yuv420 - - id: camera2 - path: dora-pyrealsense - _unstable_deploy: - machine: encoder - inputs: - tick: dora/timer/millis/10 - outputs: - - image - env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - ENCODING: jpeg - id: echo build: pip install -e ../../node-hub/dora-echo path: dora-echo _unstable_deploy: machine: decoder - inputs: - image: camera2/image + #inputs: + # image: camera/image outputs: - image - id: rav1e-local - path: dora-av1-encoder - build: cargo build -p dora-av1-encoder --release + path: dora-rav1e + build: cargo build -p dora-rav1e --release _unstable_deploy: machine: encoder - #inputs: - # image: camera/image + inputs: + image: camera/image outputs: - - frame + - image env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 @@ -55,22 +42,22 @@ nodes: _unstable_deploy: machine: decoder inputs: - image: av-local/frame + image: rav1e-local/image outputs: - - frame + - image env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - id: rav1e-remote - path: dora-av1-encoder - build: cargo build -p dora-av1-encoder --release + path: dora-rav1e + build: cargo build -p dora-rav1e --release _unstable_deploy: machine: decoder inputs: image: dav1d-remote/image outputs: - - frame + - image env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 @@ -79,11 +66,11 @@ nodes: path: dora-dav1d build: cargo build -p dora-dav1d --release _unstable_deploy: - machine: decoder + machine: encoder inputs: - image: rav1e-local/frame + image: rav1e-remote/image outputs: - - frame + - image env: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 @@ -94,14 +81,14 @@ nodes: machine: encoder path: dora-rerun inputs: - image: dav1d-local/frame - # image: echo/image + image_decode: dav1d-local/image + image_echo: echo/image #image_2: camera6/image - - id: plot2 - build: pip install -e ../../node-hub/opencv-plot - _unstable_deploy: - machine: encoder - path: opencv-plot - inputs: - image: echo/image + # - id: plot2 + # build: pip install -e ../../node-hub/opencv-plot + # _unstable_deploy: + # machine: encoder + # path: opencv-plot + # inputs: + # image: echo/image diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index 0dbe2106..44f0f55c 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -41,7 +41,6 @@ fn main() -> Result<()> { let mut settings = Settings::new(); // settings.set_n_threads(16); settings.set_max_frame_delay(1); - let mut dec = dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance"); @@ -78,17 +77,34 @@ fn main() -> Result<()> { } else { 480 }; - let y = p.plane(dav1d::PlanarImageComponent::Y); - let u = p.plane(dav1d::PlanarImageComponent::U); - let v = p.plane(dav1d::PlanarImageComponent::V); - let y = yuv420_to_bgr(&y, &u, &v, width, height); - - let arrow = y.into_arrow(); - metadata.parameters.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("bgr8".to_string()), - ); - node.send_output(id, metadata.parameters, arrow).unwrap(); + match p.pixel_layout() { + dav1d::PixelLayout::I420 => { + let y = p.plane(dav1d::PlanarImageComponent::Y); + let u = p.plane(dav1d::PlanarImageComponent::U); + let v = p.plane(dav1d::PlanarImageComponent::V); + let y = yuv420_to_bgr(&y, &u, &v, width, height); + let arrow = y.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("bgr8".to_string()), + ); + node.send_output(id, metadata.parameters, arrow).unwrap(); + } + dav1d::PixelLayout::I400 => { + let y = p.plane(dav1d::PlanarImageComponent::Y); + let vec16: Vec = bytemuck::cast_slice(&y).to_vec(); + let arrow = vec16.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("mono16".to_string()), + ); + node.send_output(id, metadata.parameters, arrow).unwrap(); + } + _ => { + warn!("Unsupported pixel layout"); + continue; + } + }; } } } diff --git a/node-hub/dora-rav1e/src/main.rs b/node-hub/dora-rav1e/src/main.rs index bff6cf33..38e114b5 100644 --- a/node-hub/dora-rav1e/src/main.rs +++ b/node-hub/dora-rav1e/src/main.rs @@ -9,7 +9,7 @@ use std::env::var; -use dora_node_api::arrow::array::UInt8Array; +use dora_node_api::arrow::array::{UInt16Array, UInt8Array}; use dora_node_api::{DoraNode, Event, IntoArrow, Parameter}; use eyre::{Context as EyreContext, Result}; use log::warn; @@ -119,6 +119,14 @@ fn main() -> Result<()> { low_latency: true, ..Default::default() }; + match encoding { + "mono16" => { + enc.bit_depth = 12; + enc.chroma_sampling = color::ChromaSampling::Cs400; + } + _ => {} + } + let cfg = Config::new().with_encoder_config(enc.clone()); if encoding == "bgr8" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); @@ -192,6 +200,48 @@ fn main() -> Result<()> { let stride = (width + xdec) >> xdec; f.planes[2].copy_from_raw_u8(&v, stride, 1); + match ctx.send_frame(f) { + Ok(_) => {} + Err(e) => match e { + EncoderStatus::EnoughData => { + warn!("Unable to send frame "); + } + _ => { + warn!("Unable to send frame "); + } + }, + } + metadata + .parameters + .insert("encoding".to_string(), Parameter::String("av1".to_string())); + ctx.flush(); + match ctx.receive_packet() { + Ok(pkt) => { + let data = pkt.data; + let arrow = data.into_arrow(); + node.send_output(id, metadata.parameters, arrow) + .context("could not send output") + .unwrap(); + } + Err(e) => match e { + EncoderStatus::LimitReached => {} + EncoderStatus::Encoded => {} + EncoderStatus::NeedMoreData => {} + _ => { + panic!("Unable to receive packet",); + } + }, + } + vec![] + } else if encoding == "mono16" { + let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap(); + let buffer: &[u16] = buffer.values(); + let mut ctx: Context = cfg.new_context().unwrap(); + let mut f = ctx.new_frame(); + + let origin = f.planes[0].data_origin_mut(); + origin.copy_from_slice(buffer); + match ctx.send_frame(f) { Ok(_) => {} Err(e) => match e { From 61fb91eb5305bcadc149e98689a506e6be0ea2ad Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 3 Apr 2025 19:06:13 +0200 Subject: [PATCH 08/26] Fix mono16 encoding --- Cargo.lock | 2 + node-hub/dora-dav1d/src/main.rs | 100 +++++++++----------- node-hub/dora-rav1e/Cargo.toml | 1 + node-hub/dora-rav1e/src/main.rs | 26 ++---- node-hub/dora-rerun/src/lib.rs | 158 +++++++++++++++++++++----------- 5 files changed, 165 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40d2d356..dbb2f973 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3178,6 +3178,7 @@ name = "dora-dav1d" version = "0.0.0" dependencies = [ "bitstream-io", + "bytemuck", "dav1d", "dora-node-api", "eyre", @@ -3436,6 +3437,7 @@ dependencies = [ name = "dora-rav1e" version = "0.3.10" dependencies = [ + "bytemuck", "dora-node-api", "eyre", "log", diff --git a/node-hub/dora-dav1d/src/main.rs b/node-hub/dora-dav1d/src/main.rs index 44f0f55c..a31b48ed 100644 --- a/node-hub/dora-dav1d/src/main.rs +++ b/node-hub/dora-dav1d/src/main.rs @@ -1,5 +1,5 @@ use dav1d::Settings; -use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow, Parameter}; +use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow}; use eyre::{Context, Result}; use log::warn; @@ -7,9 +7,11 @@ fn yuv420_to_bgr( y_plane: &[u8], u_plane: &[u8], v_plane: &[u8], - width: usize, - height: usize, + width: u32, + height: u32, ) -> Vec { + let width = width as usize; + let height = height as usize; let mut rgb_data = vec![0u8; width * height * 3]; // Output RGB data buffer for j in 0..height { @@ -54,65 +56,53 @@ fn main() -> Result<()> { data, mut metadata, }) => { - let data = data.as_any().downcast_ref::().unwrap(); - let data = data.values().clone(); - - match dec.send_data(data, None, None, None) { - Err(e) => { - warn!("Error sending data to the decoder: {}", e); - } - Ok(()) => { - if let Ok(p) = dec.get_picture() { - let height = if let Some(Parameter::Integer(h)) = - metadata.parameters.get("height") - { - *h as usize - } else { - 640 - }; - let width = if let Some(Parameter::Integer(w)) = - metadata.parameters.get("width") - { - *w as usize - } else { - 480 - }; - match p.pixel_layout() { - dav1d::PixelLayout::I420 => { - let y = p.plane(dav1d::PlanarImageComponent::Y); - let u = p.plane(dav1d::PlanarImageComponent::U); - let v = p.plane(dav1d::PlanarImageComponent::V); - let y = yuv420_to_bgr(&y, &u, &v, width, height); - let arrow = y.into_arrow(); - metadata.parameters.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("bgr8".to_string()), - ); - node.send_output(id, metadata.parameters, arrow).unwrap(); - } - dav1d::PixelLayout::I400 => { - let y = p.plane(dav1d::PlanarImageComponent::Y); - let vec16: Vec = bytemuck::cast_slice(&y).to_vec(); - let arrow = vec16.into_arrow(); - metadata.parameters.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("mono16".to_string()), - ); - node.send_output(id, metadata.parameters, arrow).unwrap(); - } - _ => { - warn!("Unsupported pixel layout"); - continue; - } - }; + if let Some(data) = data.as_any().downcast_ref::() { + let data = data.values().clone(); + match dec.send_data(data, None, None, None) { + Err(e) => { + warn!("Error sending data to the decoder: {}", e); + } + Ok(()) => { + if let Ok(p) = dec.get_picture() { + match p.pixel_layout() { + dav1d::PixelLayout::I420 => { + let y = p.plane(dav1d::PlanarImageComponent::Y); + let u = p.plane(dav1d::PlanarImageComponent::U); + let v = p.plane(dav1d::PlanarImageComponent::V); + let y = yuv420_to_bgr(&y, &u, &v, p.width(), p.height()); + let arrow = y.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("bgr8".to_string()), + ); + node.send_output(id, metadata.parameters, arrow).unwrap(); + } + dav1d::PixelLayout::I400 => { + let y = p.plane(dav1d::PlanarImageComponent::Y); + let vec16: Vec = bytemuck::cast_slice(&y).to_vec(); + let arrow = vec16.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String("mono16".to_string()), + ); + node.send_output(id, metadata.parameters, arrow).unwrap(); + } + _ => { + warn!("Unsupported pixel layout"); + continue; + } + }; + } } } + } else { + warn!("Unsupported data type {}", data.data_type()); + continue; } } None => break, Some(_) => break, } } - Ok(()) } diff --git a/node-hub/dora-rav1e/Cargo.toml b/node-hub/dora-rav1e/Cargo.toml index 10a6bcda..cdd4249e 100644 --- a/node-hub/dora-rav1e/Cargo.toml +++ b/node-hub/dora-rav1e/Cargo.toml @@ -14,3 +14,4 @@ rav1e = { version = "0.7.1", features = ["serialize"] } dora-node-api = { workspace = true, features = ["tracing"] } eyre = "0.6.8" log = "0.4" +bytemuck = "1.20" diff --git a/node-hub/dora-rav1e/src/main.rs b/node-hub/dora-rav1e/src/main.rs index 38e114b5..91e37973 100644 --- a/node-hub/dora-rav1e/src/main.rs +++ b/node-hub/dora-rav1e/src/main.rs @@ -93,7 +93,7 @@ fn main() -> Result<()> { DoraNode::init_from_env().context("Could not initialize dora node")?; loop { - let _buffer = match events.recv() { + match events.recv() { Some(Event::Input { id, data, @@ -154,9 +154,11 @@ fn main() -> Result<()> { Err(e) => match e { EncoderStatus::EnoughData => { warn!("Unable to send frame "); + continue; } _ => { warn!("Unable to send frame "); + continue; } }, } @@ -181,7 +183,6 @@ fn main() -> Result<()> { } }, } - vec![] } else if encoding == "yuv420" { let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); let buffer = buffer.values(); //.to_vec(); @@ -232,15 +233,19 @@ fn main() -> Result<()> { } }, } - vec![] } else if encoding == "mono16" { let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap(); let buffer: &[u16] = buffer.values(); + // let buffer = shift_u16_slice_to_upper_12_bits(buffer); + let bytes: &[u8] = &bytemuck::cast_slice(&buffer); + let mut ctx: Context = cfg.new_context().unwrap(); let mut f = ctx.new_frame(); - let origin = f.planes[0].data_origin_mut(); - origin.copy_from_slice(buffer); + let xdec = f.planes[0].cfg.xdec; + let stride = (width + xdec) >> xdec; + // Multiply by 2 the stride as it is going to be width * 2 as we're converting 16-bit to 2*8-bit. + f.planes[0].copy_from_raw_u8(bytes, stride * 2, 2); match ctx.send_frame(f) { Ok(_) => {} @@ -274,19 +279,8 @@ fn main() -> Result<()> { } }, } - vec![] } else if encoding == "rgb8" { unimplemented!("We haven't worked on additional encodings."); - let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); - let buffer: &[u8] = buffer.values(); - let mut ctx: Context = cfg.new_context().unwrap(); - let mut f = ctx.new_frame(); - - for p in &mut f.planes { - let stride = (enc.width + p.cfg.xdec) >> p.cfg.xdec; - p.copy_from_raw_u8(&buffer, stride, 1); - } - buffer.to_vec() } else { unimplemented!("We haven't worked on additional encodings."); } diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index c79d39c8..fa0b9520 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -3,10 +3,8 @@ use std::{collections::HashMap, env::VarError, path::Path}; use dora_node_api::{ - arrow::{ - array::{Array, AsArray, Float32Array, Float64Array, StringArray, UInt8Array}, - datatypes::Float32Type, - }, + arrow::array::{Array, Float32Array, Float64Array, StringArray, UInt16Array, UInt8Array}, + arrow::{array::AsArray, datatypes::Float32Type}, dora_core::config::DataId, DoraNode, Event, Parameter, }; @@ -181,54 +179,112 @@ pub fn lib_main() -> Result<()> { } else { vec![640, 480] }; - let buffer: &Float64Array = data.as_any().downcast_ref().unwrap(); - let points_3d = buffer.iter().enumerate().map(|(i, z)| { - let u = i as f32 % *width as f32; // Calculate x-coordinate (u) - let v = i as f32 / *width as f32; // Calculate y-coordinate (v) - let z = z.unwrap_or_default() as f32; + match data.data_type() { + dora_node_api::arrow::datatypes::DataType::Float64 => { + let buffer: &Float64Array = data.as_any().downcast_ref().unwrap(); - ( - (u - resolution[0] as f32) * z / focal_length[0] as f32, - (v - resolution[1] as f32) * z / focal_length[1] as f32, - z, - ) - }); - let points_3d = Points3D::new(points_3d); - if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { - let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "mask")) { - let mask_length = color_buffer.len() / 3; - let number_masks = mask.len() / mask_length; - color_buffer - .chunks(3) - .enumerate() - .map(|(e, x)| { - for i in 0..number_masks { - if mask[i * mask_length + e] && (e % 3 == 0) { - if i == 0 { - return rerun::Color::from_rgb(255, x[1], x[2]); - } else if i == 1 { - return rerun::Color::from_rgb(x[0], 255, x[2]); - } else if i == 2 { - return rerun::Color::from_rgb(x[0], x[1], 255); - } else { - return rerun::Color::from_rgb(x[0], 255, x[2]); + let points_3d = buffer.iter().enumerate().map(|(i, z)| { + let u = i as f32 % *width as f32; // Calculate x-coordinate (u) + let v = i as f32 / *width as f32; // Calculate y-coordinate (v) + let z = z.unwrap_or_default() as f32; + + ( + (u - resolution[0] as f32) * z / focal_length[0] as f32, + (v - resolution[1] as f32) * z / focal_length[1] as f32, + z, + ) + }); + let points_3d = Points3D::new(points_3d); + if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { + let colors = if let Some(mask) = + mask_cache.get(&id.replace("depth", "masks")) + { + let mask_length = color_buffer.len() / 3; + let number_masks = mask.len() / mask_length; + color_buffer + .chunks(3) + .enumerate() + .map(|(e, x)| { + for i in 0..number_masks { + if mask[i * mask_length + e] && (e % 3 == 0) { + if i == 0 { + return rerun::Color::from_rgb(255, x[1], x[2]); + } else if i == 1 { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } else if i == 2 { + return rerun::Color::from_rgb(x[0], x[1], 255); + } else { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } + } } - } - } - rerun::Color::from_rgb(x[0], x[1], x[2]) - }) - .collect::>() - } else { - color_buffer - .chunks(3) - .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) - .collect::>() - }; - rec.log(id.as_str(), &points_3d.with_colors(colors)) - .context("could not log points")?; - } else { - rec.log(id.as_str(), &points_3d) - .context("could not log points")?; + rerun::Color::from_rgb(x[0], x[1], x[2]) + }) + .collect::>() + } else { + color_buffer + .chunks(3) + .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) + .collect::>() + }; + rec.log(id.as_str(), &points_3d.with_colors(colors)) + .context("could not log points")?; + } + } + dora_node_api::arrow::datatypes::DataType::UInt16 => { + let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap(); + + let points_3d = buffer.iter().enumerate().map(|(i, z)| { + let u = i as f32 % *width as f32; // Calculate x-coordinate (u) + let v = i as f32 / *width as f32; // Calculate y-coordinate (v) + let z = z.unwrap_or_default() as f32 / 1_000.; + + ( + (u - resolution[0] as f32) * z / focal_length[0] as f32, + (v - resolution[1] as f32) * z / focal_length[1] as f32, + z, + ) + }); + let points_3d = Points3D::new(points_3d); + if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { + let colors = if let Some(mask) = + mask_cache.get(&id.replace("depth", "masks")) + { + let mask_length = color_buffer.len() / 3; + let number_masks = mask.len() / mask_length; + color_buffer + .chunks(3) + .enumerate() + .map(|(e, x)| { + for i in 0..number_masks { + if mask[i * mask_length + e] && (e % 3 == 0) { + if i == 0 { + return rerun::Color::from_rgb(255, x[1], x[2]); + } else if i == 1 { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } else if i == 2 { + return rerun::Color::from_rgb(x[0], x[1], 255); + } else { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } + } + } + rerun::Color::from_rgb(x[0], x[1], x[2]) + }) + .collect::>() + } else { + color_buffer + .chunks(3) + .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) + .collect::>() + }; + rec.log(id.as_str(), &points_3d.with_colors(colors)) + .context("could not log points")?; + } + } + _ => { + return Err(eyre!("Unsupported depth data type {}", data.data_type())); + } } } else if id.as_str().contains("text") { let buffer: StringArray = data.to_data().into(); @@ -242,7 +298,7 @@ pub fn lib_main() -> Result<()> { })?; } else if id.as_str().contains("boxes2d") { boxes2d::update_boxes2d(&rec, id, data, metadata).context("update boxes 2d")?; - } else if id.as_str().contains("mask") { + } else if id.as_str().contains("masks") { let masks = if let Some(data) = data.as_primitive_opt::() { let data = data .iter() From 548d884b9485ce9dde9552c87cbf2573bbe86037 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 4 Apr 2025 11:10:33 +0200 Subject: [PATCH 09/26] Add cosinus sinus correction of depth frame --- node-hub/dora-rerun/src/lib.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index fa0b9520..ecf7da6c 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -94,6 +94,13 @@ pub fn lib_main() -> Result<()> { } Err(VarError::NotPresent) => (), }; + let camera_pitch = std::env::var("CAMERA_PITCH") + .unwrap_or("2.47".to_string()) + .parse::() + .unwrap(); + let cos_theta = camera_pitch.cos(); // np.cos(np.deg2rad(180-38)) + let sin_theta = camera_pitch.sin(); // np.sin(np.deg2rad(180-38)) + // (0.32489833, -0.25068134, 0.4761387) while let Some(event) = events.recv() { if let Event::Input { id, data, metadata } = event { @@ -238,12 +245,13 @@ pub fn lib_main() -> Result<()> { let u = i as f32 % *width as f32; // Calculate x-coordinate (u) let v = i as f32 / *width as f32; // Calculate y-coordinate (v) let z = z.unwrap_or_default() as f32 / 1_000.; + let y = (u - resolution[0] as f32) * z / focal_length[0] as f32; + let x = (v - resolution[1] as f32) * z / focal_length[1] as f32; + let new_x = sin_theta * z + cos_theta * x; + let new_y = -y; + let new_z = cos_theta * z - sin_theta * x; - ( - (u - resolution[0] as f32) * z / focal_length[0] as f32, - (v - resolution[1] as f32) * z / focal_length[1] as f32, - z, - ) + (new_x, new_y, new_z) }); let points_3d = Points3D::new(points_3d); if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { From 8a57b9c79d7209c7ba503d1df5203f3a203b0611 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 4 Apr 2025 14:18:09 +0200 Subject: [PATCH 10/26] Adding documentation and better support for pointcloud in dora-rerun --- binaries/daemon/src/lib.rs | 2 + .../videostream-encoding}/dataflow.yml | 31 +-------- .../videostream-encoding/dataflow_reachy.yml | 68 +++++++++++++++++++ node-hub/dora-rerun/src/lib.rs | 60 ++++++++++------ 4 files changed, 114 insertions(+), 47 deletions(-) rename {node-hub/dora-dav1d => examples/videostream-encoding}/dataflow.yml (70%) create mode 100644 examples/videostream-encoding/dataflow_reachy.yml diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 474e5a29..8fe90294 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -277,6 +277,8 @@ impl Daemon { r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#, ) .unwrap(); + + // Linkstate make it possible to connect two daemons on different network through a public daemon zenoh_config .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) .unwrap(); diff --git a/node-hub/dora-dav1d/dataflow.yml b/examples/videostream-encoding/dataflow.yml similarity index 70% rename from node-hub/dora-dav1d/dataflow.yml rename to examples/videostream-encoding/dataflow.yml index 068ecd76..131da27a 100644 --- a/node-hub/dora-dav1d/dataflow.yml +++ b/examples/videostream-encoding/dataflow.yml @@ -9,19 +9,9 @@ nodes: outputs: - image env: - CAPTURE_PATH: 1 - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - - - id: echo - build: pip install -e ../../node-hub/dora-echo - path: dora-echo - _unstable_deploy: - machine: decoder - #inputs: - # image: camera/image - outputs: - - image + CAPTURE_PATH: 0 + IMAGE_WIDTH: 1280 + IMAGE_HEIGHT: 720 - id: rav1e-local path: dora-rav1e @@ -32,9 +22,6 @@ nodes: image: camera/image outputs: - image - env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - id: dav1d-remote path: dora-dav1d @@ -45,9 +32,6 @@ nodes: image: rav1e-local/image outputs: - image - env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - id: rav1e-remote path: dora-rav1e @@ -83,12 +67,3 @@ nodes: inputs: image_decode: dav1d-local/image image_echo: echo/image - #image_2: camera6/image - - # - id: plot2 - # build: pip install -e ../../node-hub/opencv-plot - # _unstable_deploy: - # machine: encoder - # path: opencv-plot - # inputs: - # image: echo/image diff --git a/examples/videostream-encoding/dataflow_reachy.yml b/examples/videostream-encoding/dataflow_reachy.yml new file mode 100644 index 00000000..0fc7f3ac --- /dev/null +++ b/examples/videostream-encoding/dataflow_reachy.yml @@ -0,0 +1,68 @@ +nodes: + - id: camera + path: dora-reachy2-camera + _unstable_deploy: + machine: encoder + inputs: + tick: dora/timer/millis/50 + outputs: + - image_right + - image_left + - image_depth + - depth + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + ROBOT_IP: 127.0.0.1 + + - id: rav1e-local + path: dora-rav1e + build: cargo build -p dora-rav1e --release + _unstable_deploy: + machine: encoder + inputs: + depth: camera/depth + outputs: + - depth + env: + RAV1E_SPEED: 7 + + - id: rav1e-local-image + path: dora-rav1e + build: cargo build -p dora-rav1e --release + _unstable_deploy: + machine: encoder + inputs: + image_depth: camera/image_depth + image_left: camera/image_left + outputs: + - image_left + - image_depth + - depth + env: + RAV1E_SPEED: 10 + + - id: dav1d-remote + path: dora-dav1d + build: cargo build -p dora-dav1d --release + _unstable_deploy: + machine: plot + inputs: + image_depth: rav1e-local-image/image_depth + image_left: rav1e-local-image/image_left + depth: rav1e-local/depth + outputs: + - image_left + - image_depth + - depth + + - id: plot + build: pip install -e ../../node-hub/dora-rerun + _unstable_deploy: + machine: plot + path: dora-rerun + inputs: + image: dav1d-remote/image_depth + depth: dav1d-remote/depth + image_left: dav1d-remote/image_left diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index ecf7da6c..1d06fe77 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -190,18 +190,30 @@ pub fn lib_main() -> Result<()> { dora_node_api::arrow::datatypes::DataType::Float64 => { let buffer: &Float64Array = data.as_any().downcast_ref().unwrap(); - let points_3d = buffer.iter().enumerate().map(|(i, z)| { + let mut points = vec![]; + buffer.iter().enumerate().for_each(|(i, z)| { let u = i as f32 % *width as f32; // Calculate x-coordinate (u) let v = i as f32 / *width as f32; // Calculate y-coordinate (v) - let z = z.unwrap_or_default() as f32; - ( - (u - resolution[0] as f32) * z / focal_length[0] as f32, - (v - resolution[1] as f32) * z / focal_length[1] as f32, - z, - ) + if let Some(z) = z { + let z = z as f32; + // Skip points that have empty depth or is too far away + if z == 0. || z > 8.0 { + points.push((0., 0., 0.)); + return; + } + let y = (u - resolution[0] as f32) * z / focal_length[0] as f32; + let x = (v - resolution[1] as f32) * z / focal_length[1] as f32; + let new_x = sin_theta * z + cos_theta * x; + let new_y = -y; + let new_z = cos_theta * z - sin_theta * x; + + points.push((new_x, new_y, new_z)); + } else { + points.push((0., 0., 0.)); + } }); - let points_3d = Points3D::new(points_3d); + let points_3d = Points3D::new(points); if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "masks")) @@ -240,20 +252,30 @@ pub fn lib_main() -> Result<()> { } dora_node_api::arrow::datatypes::DataType::UInt16 => { let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap(); - - let points_3d = buffer.iter().enumerate().map(|(i, z)| { + let mut points = vec![]; + buffer.iter().enumerate().for_each(|(i, z)| { let u = i as f32 % *width as f32; // Calculate x-coordinate (u) let v = i as f32 / *width as f32; // Calculate y-coordinate (v) - let z = z.unwrap_or_default() as f32 / 1_000.; - let y = (u - resolution[0] as f32) * z / focal_length[0] as f32; - let x = (v - resolution[1] as f32) * z / focal_length[1] as f32; - let new_x = sin_theta * z + cos_theta * x; - let new_y = -y; - let new_z = cos_theta * z - sin_theta * x; - - (new_x, new_y, new_z) + + if let Some(z) = z { + let z = z as f32; + // Skip points that have empty depth or is too far away + if z == 0. || z > 8.0 { + points.push((0., 0., 0.)); + return; + } + let y = (u - resolution[0] as f32) * z / focal_length[0] as f32; + let x = (v - resolution[1] as f32) * z / focal_length[1] as f32; + let new_x = sin_theta * z + cos_theta * x; + let new_y = -y; + let new_z = cos_theta * z - sin_theta * x; + + points.push((new_x, new_y, new_z)); + } else { + points.push((0., 0., 0.)); + } }); - let points_3d = Points3D::new(points_3d); + let points_3d = Points3D::new(points); if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "masks")) From d824c6a15a5f281c94ea2a91d997e8321d74d609 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Fri, 4 Apr 2025 14:33:24 +0200 Subject: [PATCH 11/26] Refactor support for masks --- node-hub/dora-rerun/src/lib.rs | 109 +++++++++++---------------------- 1 file changed, 36 insertions(+), 73 deletions(-) diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index 1d06fe77..46ce4fd1 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -186,7 +186,7 @@ pub fn lib_main() -> Result<()> { } else { vec![640, 480] }; - match data.data_type() { + let points = match data.data_type() { dora_node_api::arrow::datatypes::DataType::Float64 => { let buffer: &Float64Array = data.as_any().downcast_ref().unwrap(); @@ -213,42 +213,7 @@ pub fn lib_main() -> Result<()> { points.push((0., 0., 0.)); } }); - let points_3d = Points3D::new(points); - if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { - let colors = if let Some(mask) = - mask_cache.get(&id.replace("depth", "masks")) - { - let mask_length = color_buffer.len() / 3; - let number_masks = mask.len() / mask_length; - color_buffer - .chunks(3) - .enumerate() - .map(|(e, x)| { - for i in 0..number_masks { - if mask[i * mask_length + e] && (e % 3 == 0) { - if i == 0 { - return rerun::Color::from_rgb(255, x[1], x[2]); - } else if i == 1 { - return rerun::Color::from_rgb(x[0], 255, x[2]); - } else if i == 2 { - return rerun::Color::from_rgb(x[0], x[1], 255); - } else { - return rerun::Color::from_rgb(x[0], 255, x[2]); - } - } - } - rerun::Color::from_rgb(x[0], x[1], x[2]) - }) - .collect::>() - } else { - color_buffer - .chunks(3) - .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) - .collect::>() - }; - rec.log(id.as_str(), &points_3d.with_colors(colors)) - .context("could not log points")?; - } + Points3D::new(points) } dora_node_api::arrow::datatypes::DataType::UInt16 => { let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap(); @@ -275,46 +240,44 @@ pub fn lib_main() -> Result<()> { points.push((0., 0., 0.)); } }); - let points_3d = Points3D::new(points); - if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { - let colors = if let Some(mask) = - mask_cache.get(&id.replace("depth", "masks")) - { - let mask_length = color_buffer.len() / 3; - let number_masks = mask.len() / mask_length; - color_buffer - .chunks(3) - .enumerate() - .map(|(e, x)| { - for i in 0..number_masks { - if mask[i * mask_length + e] && (e % 3 == 0) { - if i == 0 { - return rerun::Color::from_rgb(255, x[1], x[2]); - } else if i == 1 { - return rerun::Color::from_rgb(x[0], 255, x[2]); - } else if i == 2 { - return rerun::Color::from_rgb(x[0], x[1], 255); - } else { - return rerun::Color::from_rgb(x[0], 255, x[2]); - } - } - } - rerun::Color::from_rgb(x[0], x[1], x[2]) - }) - .collect::>() - } else { - color_buffer - .chunks(3) - .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) - .collect::>() - }; - rec.log(id.as_str(), &points_3d.with_colors(colors)) - .context("could not log points")?; - } + Points3D::new(points) } _ => { return Err(eyre!("Unsupported depth data type {}", data.data_type())); } + }; + if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) { + let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "masks")) { + let mask_length = color_buffer.len() / 3; + let number_masks = mask.len() / mask_length; + color_buffer + .chunks(3) + .enumerate() + .map(|(e, x)| { + for i in 0..number_masks { + if mask[i * mask_length + e] && (e % 3 == 0) { + if i == 0 { + return rerun::Color::from_rgb(255, x[1], x[2]); + } else if i == 1 { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } else if i == 2 { + return rerun::Color::from_rgb(x[0], x[1], 255); + } else { + return rerun::Color::from_rgb(x[0], 255, x[2]); + } + } + } + rerun::Color::from_rgb(x[0], x[1], x[2]) + }) + .collect::>() + } else { + color_buffer + .chunks(3) + .map(|x| rerun::Color::from_rgb(x[0], x[1], x[2])) + .collect::>() + }; + rec.log(id.as_str(), &points.with_colors(colors)) + .context("could not log points")?; } } else if id.as_str().contains("text") { let buffer: StringArray = data.to_data().into(); From 82c5ee21078059025eb76b14704fbb90ce01c345 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Fri, 4 Apr 2025 15:19:16 +0200 Subject: [PATCH 12/26] Make camera pitch parametrisation easier --- .../dora-ios-lidar/dora_ios_lidar/main.py | 101 +++++++++++------- node-hub/dora-rerun/src/lib.rs | 18 ++-- 2 files changed, 73 insertions(+), 46 deletions(-) diff --git a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py index af6c9382..2615e61b 100644 --- a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py +++ b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py @@ -7,6 +7,7 @@ import numpy as np import pyarrow as pa from dora import Node from record3d import Record3DStream +from scipy.spatial.transform import Rotation class DemoApp: @@ -54,6 +55,21 @@ class DemoApp: [[coeffs.fx, 0, coeffs.tx], [0, coeffs.fy, coeffs.ty], [0, 0, 1]], ) + def get_camera_pose(self): + """Get Camera Pose.""" + pose = self.session.get_camera_pose() + rot = Rotation.from_quat([pose.qx, pose.qy, pose.qz, pose.qw]) + euler = rot.as_euler("xyz", degrees=False) + + return [ + pose.tx, + pose.ty, + pose.tz, + pose.qx, + euler[1], + euler[2], + ] + def start_processing_stream(self): """TODO: Add docstring.""" node = Node() @@ -61,47 +77,52 @@ class DemoApp: for event in node: if self.stop: break + if event["type"] == "INPUT": - if event["id"] == "TICK": - self.event.wait() # Wait for new frame to arrive - - # Copy the newly arrived RGBD frame - depth = self.session.get_depth_frame() - rgb = self.session.get_rgb_frame() - intrinsic_mat = self.get_intrinsic_mat_from_coeffs( - self.session.get_intrinsic_mat(), - ) - - if depth.shape != rgb.shape: - rgb = cv2.resize(rgb, (depth.shape[1], depth.shape[0])) - - node.send_output( - "image", - pa.array(rgb.ravel()), - metadata={ - "encoding": "rgb8", - "width": rgb.shape[1], - "height": rgb.shape[0], - }, - ) - - node.send_output( - "depth", - pa.array(depth.ravel().astype(np.float64())), - metadata={ - "width": depth.shape[1], - "height": depth.shape[0], - "encoding": "CV_64F", - "focal": [ - int(intrinsic_mat[0, 0]), - int(intrinsic_mat[1, 1]), - ], - "resolution": [ - int(intrinsic_mat[0, 2]), - int(intrinsic_mat[1, 2]), - ], - }, - ) + self.event.wait() # Wait for new frame to arrive + + # Copy the newly arrived RGBD frame + depth = self.session.get_depth_frame() + rgb = self.session.get_rgb_frame() + intrinsic_mat = self.get_intrinsic_mat_from_coeffs( + self.session.get_intrinsic_mat(), + ) + pose = self.get_camera_pose() + + if depth.shape != rgb.shape: + rgb = cv2.resize(rgb, (depth.shape[1], depth.shape[0])) + node.send_output( + "image", + pa.array(rgb.ravel()), + metadata={ + "encoding": "rgb8", + "width": rgb.shape[1], + "height": rgb.shape[0], + }, + ) + + depth = (np.array(depth) * 1_000).astype(np.uint16) + + node.send_output( + "depth", + pa.array(depth.ravel()), + metadata={ + "width": depth.shape[1], + "height": depth.shape[0], + "encoding": "mono16", + "focal": [ + int(intrinsic_mat[0, 0]), + int(intrinsic_mat[1, 1]), + ], + "resolution": [ + int(intrinsic_mat[0, 2]), + int(intrinsic_mat[1, 2]), + ], + "roll": pose[3], + "pitch": pose[4], + "yaw": pose[5], + }, + ) self.event.clear() diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index 46ce4fd1..c482e718 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -95,12 +95,9 @@ pub fn lib_main() -> Result<()> { Err(VarError::NotPresent) => (), }; let camera_pitch = std::env::var("CAMERA_PITCH") - .unwrap_or("2.47".to_string()) + .unwrap_or("0.0".to_string()) .parse::() .unwrap(); - let cos_theta = camera_pitch.cos(); // np.cos(np.deg2rad(180-38)) - let sin_theta = camera_pitch.sin(); // np.sin(np.deg2rad(180-38)) - // (0.32489833, -0.25068134, 0.4761387) while let Some(event) = events.recv() { if let Event::Input { id, data, metadata } = event { @@ -186,6 +183,15 @@ pub fn lib_main() -> Result<()> { } else { vec![640, 480] }; + let pitch = if let Some(Parameter::Float(pitch)) = metadata.parameters.get("pitch") + { + *pitch as f32 + } else { + camera_pitch + }; + let cos_theta = pitch.cos(); // np.cos(np.deg2rad(180-38)) + let sin_theta = pitch.sin(); // np.sin(np.deg2rad(180-38)) + let points = match data.data_type() { dora_node_api::arrow::datatypes::DataType::Float64 => { let buffer: &Float64Array = data.as_any().downcast_ref().unwrap(); @@ -223,8 +229,8 @@ pub fn lib_main() -> Result<()> { let v = i as f32 / *width as f32; // Calculate y-coordinate (v) if let Some(z) = z { - let z = z as f32; - // Skip points that have empty depth or is too far away + let z = z as f32 / 1000.0; // Convert to meters + // Skip points that have empty depth or is too far away if z == 0. || z > 8.0 { points.push((0., 0., 0.)); return; From 0672748b8390c821e7bcef9208eed449ad40de13 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Fri, 4 Apr 2025 15:43:10 +0200 Subject: [PATCH 13/26] Fix CI/CD by escaping dav1d when possible --- .github/workflows/ci.yml | 22 +++++++++++----------- .github/workflows/node-hub-ci-cd.yml | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 928b81f6..d12e463d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - name: Free disk Space (Windows) if: runner.os == 'Windows' run: | - docker system prune --all -f + docker system prune --all --exclude dora-dav1d -f Remove-Item "C:\Android" -Force -Recurse - uses: Swatinem/rust-cache@v2 with: @@ -63,11 +63,11 @@ jobs: cache-directories: ${{ env.CARGO_TARGET_DIR }} - name: "Check" - run: cargo check --all + run: cargo check --all --exclude dora-dav1d - name: "Build (Without Python dep as it is build with maturin)" - run: cargo build --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + run: cargo build --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python - name: "Test" - run: cargo test --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + run: cargo test --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python # Run examples as separate job because otherwise we will exhaust the disk # space of the GitHub action runners. @@ -102,7 +102,7 @@ jobs: - name: Free disk Space (Windows) if: runner.os == 'Windows' run: | - docker system prune --all -f + docker system prune --all --exclude dora-dav1d -f Remove-Item "C:\Android" -Force -Recurse - uses: Swatinem/rust-cache@v2 with: @@ -307,7 +307,7 @@ jobs: # Test Rust template Project dora new test_rust_project --internal-create-with-path-dependencies cd test_rust_project - cargo build --all + cargo build --all --exclude dora-dav1d dora up dora list dora start dataflow.yml --name ci-rust-test --detach @@ -447,12 +447,12 @@ jobs: - run: cargo --version --verbose - name: "Clippy" - run: cargo clippy --all + run: cargo clippy --all --exclude dora-dav1d - name: "Clippy (tracing feature)" - run: cargo clippy --all --features tracing + run: cargo clippy --all --exclude dora-dav1d --features tracing if: false # only the dora-runtime has this feature, but it is currently commented out - name: "Clippy (metrics feature)" - run: cargo clippy --all --features metrics + run: cargo clippy --all --exclude dora-dav1d --features metrics if: false # only the dora-runtime has this feature, but it is currently commented out rustfmt: @@ -462,7 +462,7 @@ jobs: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 - name: "rustfmt" - run: cargo fmt --all -- --check + run: cargo fmt --all --exclude dora-dav1d -- --check check-license: name: "License Checks" @@ -532,4 +532,4 @@ jobs: with: use-cross: true command: check - args: --target ${{ matrix.platform.target }} --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + args: --target ${{ matrix.platform.target }} --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml index 39f901ec..1248c36d 100644 --- a/.github/workflows/node-hub-ci-cd.yml +++ b/.github/workflows/node-hub-ci-cd.yml @@ -53,6 +53,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get install portaudio19-dev + sudo apt-get install libdav1d-dev # Install mingw-w64 cross-compilers sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 @@ -60,6 +61,7 @@ jobs: if: runner.os == 'MacOS' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))) run: | brew install portaudio + brew install dav1d - name: Set up Python if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) From d3ddc94a11e90e6e9dd09ce6adf1afa0cc452f03 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Fri, 4 Apr 2025 15:49:55 +0200 Subject: [PATCH 14/26] Escape rav1e and install nasm dependencies --- .github/workflows/ci.yml | 22 +++++++++++----------- .github/workflows/node-hub-ci-cd.yml | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d12e463d..85323d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - name: Free disk Space (Windows) if: runner.os == 'Windows' run: | - docker system prune --all --exclude dora-dav1d -f + docker system prune --all -f Remove-Item "C:\Android" -Force -Recurse - uses: Swatinem/rust-cache@v2 with: @@ -63,11 +63,11 @@ jobs: cache-directories: ${{ env.CARGO_TARGET_DIR }} - name: "Check" - run: cargo check --all --exclude dora-dav1d + run: cargo check --all --exclude dora-dav1d --exclude dora-rav1e - name: "Build (Without Python dep as it is build with maturin)" - run: cargo build --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + run: cargo build --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python - name: "Test" - run: cargo test --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + run: cargo test --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python # Run examples as separate job because otherwise we will exhaust the disk # space of the GitHub action runners. @@ -102,7 +102,7 @@ jobs: - name: Free disk Space (Windows) if: runner.os == 'Windows' run: | - docker system prune --all --exclude dora-dav1d -f + docker system prune --all -f Remove-Item "C:\Android" -Force -Recurse - uses: Swatinem/rust-cache@v2 with: @@ -307,7 +307,7 @@ jobs: # Test Rust template Project dora new test_rust_project --internal-create-with-path-dependencies cd test_rust_project - cargo build --all --exclude dora-dav1d + cargo build --all --exclude dora-dav1d --exclude dora-rav1e dora up dora list dora start dataflow.yml --name ci-rust-test --detach @@ -447,12 +447,12 @@ jobs: - run: cargo --version --verbose - name: "Clippy" - run: cargo clippy --all --exclude dora-dav1d + run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e - name: "Clippy (tracing feature)" - run: cargo clippy --all --exclude dora-dav1d --features tracing + run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features tracing if: false # only the dora-runtime has this feature, but it is currently commented out - name: "Clippy (metrics feature)" - run: cargo clippy --all --exclude dora-dav1d --features metrics + run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features metrics if: false # only the dora-runtime has this feature, but it is currently commented out rustfmt: @@ -462,7 +462,7 @@ jobs: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 - name: "rustfmt" - run: cargo fmt --all --exclude dora-dav1d -- --check + run: cargo fmt --all --exclude dora-dav1d --exclude dora-rav1e -- --check check-license: name: "License Checks" @@ -532,4 +532,4 @@ jobs: with: use-cross: true command: check - args: --target ${{ matrix.platform.target }} --all --exclude dora-dav1d --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python + args: --target ${{ matrix.platform.target }} --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml index 1248c36d..68b141ab 100644 --- a/.github/workflows/node-hub-ci-cd.yml +++ b/.github/workflows/node-hub-ci-cd.yml @@ -53,7 +53,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get install portaudio19-dev - sudo apt-get install libdav1d-dev + sudo apt-get install libdav1d-dev nasm # Install mingw-w64 cross-compilers sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 @@ -61,7 +61,7 @@ jobs: if: runner.os == 'MacOS' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))) run: | brew install portaudio - brew install dav1d + brew install dav1d nasm - name: Set up Python if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) From dbac146c9b0709e209001d1aa4da2b9624267e79 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 14:50:39 +0200 Subject: [PATCH 15/26] Fix CI/CD --- .github/workflows/ci.yml | 2 +- binaries/daemon/src/lib.rs | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85323d69..d80f04d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -462,7 +462,7 @@ jobs: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 - name: "rustfmt" - run: cargo fmt --all --exclude dora-dav1d --exclude dora-rav1e -- --check + run: cargo fmt --all -- --check check-license: name: "License Checks" diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 8fe90294..d71fe152 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -265,12 +265,6 @@ impl Daemon { let mut zenoh_config = zenoh::Config::default(); if let Some(addr) = coordinator_addr { - zenoh_config - .insert_json5( - "connect/endpoints", - &format!(r#"["tcp/{}:5456"]"#, addr.ip()), - ) - .unwrap(); zenoh_config .insert_json5( "listen/endpoints", @@ -282,11 +276,22 @@ impl Daemon { zenoh_config .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) .unwrap(); - if cfg!(not(target_os = "linux")) { - warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); - zenoh_config - .insert_json5("scouting/multicast", r#"{ enabled: false }"#) - .unwrap(); + zenoh_config + .insert_json5( + "connect/endpoints", + &format!( + r#"{{ router: ["tcp/[::]:7447"], peer: ["tcp/{}:5456"] }}"#, + addr.ip() + ), + ) + .unwrap(); + if !addr.ip().is_loopback() { + if cfg!(not(target_os = "linux")) { + warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); + zenoh_config + .insert_json5("scouting/multicast", r#"{ enabled: false }"#) + .unwrap(); + } } }; if let Ok(zenoh_session) = zenoh::open(zenoh_config).await { From e3de2f9d27a44fd329f2b4532153e9d16dd76219 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 14:50:48 +0200 Subject: [PATCH 16/26] Add support for rgb8 --- node-hub/dora-rav1e/src/main.rs | 56 ++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/node-hub/dora-rav1e/src/main.rs b/node-hub/dora-rav1e/src/main.rs index 91e37973..076bee23 100644 --- a/node-hub/dora-rav1e/src/main.rs +++ b/node-hub/dora-rav1e/src/main.rs @@ -280,7 +280,61 @@ fn main() -> Result<()> { }, } } else if encoding == "rgb8" { - unimplemented!("We haven't worked on additional encodings."); + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: Vec = buffer.values().to_vec(); + let buffer: Vec = + buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect(); + let (y, u, v) = bgr8_to_yuv420(buffer, width, height); + + // Transpose values from BGR to RGB + + let mut ctx: Context = cfg.new_context().unwrap(); + let mut f = ctx.new_frame(); + + let xdec = f.planes[0].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[0].copy_from_raw_u8(&y, stride, 1); + let xdec = f.planes[1].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[1].copy_from_raw_u8(&u, stride, 1); + let xdec = f.planes[2].cfg.xdec; + let stride = (width + xdec) >> xdec; + f.planes[2].copy_from_raw_u8(&v, stride, 1); + + match ctx.send_frame(f) { + Ok(_) => {} + Err(e) => match e { + EncoderStatus::EnoughData => { + warn!("Unable to send frame "); + continue; + } + _ => { + warn!("Unable to send frame "); + continue; + } + }, + } + metadata + .parameters + .insert("encoding".to_string(), Parameter::String("av1".to_string())); + ctx.flush(); + match ctx.receive_packet() { + Ok(pkt) => { + let data = pkt.data; + let arrow = data.into_arrow(); + node.send_output(id, metadata.parameters, arrow) + .context("could not send output") + .unwrap(); + } + Err(e) => match e { + EncoderStatus::LimitReached => {} + EncoderStatus::Encoded => {} + EncoderStatus::NeedMoreData => {} + _ => { + panic!("Unable to receive packet",); + } + }, + } } else { unimplemented!("We haven't worked on additional encodings."); } From a6fe69cbed3eec81b4980e7f2522d8638442f6c3 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 16:39:15 +0200 Subject: [PATCH 17/26] fix ci/cd and improve examples --- Cargo.lock | 4 +- .../dataflow.yml | 0 .../dataflow_reachy.yml | 0 examples/av1-encoding/ios-dev.yaml | 73 +++++++++++++++++++ examples/depth_camera/ios-dev.yaml | 3 + .../dora-ios-lidar/dora_ios_lidar/main.py | 35 ++++++--- node-hub/dora-ios-lidar/pyproject.toml | 4 +- node-hub/dora-rerun/src/lib.rs | 4 +- 8 files changed, 108 insertions(+), 15 deletions(-) rename examples/{videostream-encoding => av1-encoding}/dataflow.yml (100%) rename examples/{videostream-encoding => av1-encoding}/dataflow_reachy.yml (100%) create mode 100644 examples/av1-encoding/ios-dev.yaml diff --git a/Cargo.lock b/Cargo.lock index dbb2f973..f932199a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2779,9 +2779,9 @@ checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "dav1d" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4b54a40baf633a71c6f0fb49494a7e4ee7bc26f3e727212b6cb915aa1ea1e1" +checksum = "80c3f80814db85397819d464bb553268992c393b4b3b5554b89c1655996d5926" dependencies = [ "av-data", "bitflags 2.9.0", diff --git a/examples/videostream-encoding/dataflow.yml b/examples/av1-encoding/dataflow.yml similarity index 100% rename from examples/videostream-encoding/dataflow.yml rename to examples/av1-encoding/dataflow.yml diff --git a/examples/videostream-encoding/dataflow_reachy.yml b/examples/av1-encoding/dataflow_reachy.yml similarity index 100% rename from examples/videostream-encoding/dataflow_reachy.yml rename to examples/av1-encoding/dataflow_reachy.yml diff --git a/examples/av1-encoding/ios-dev.yaml b/examples/av1-encoding/ios-dev.yaml new file mode 100644 index 00000000..d14fdb36 --- /dev/null +++ b/examples/av1-encoding/ios-dev.yaml @@ -0,0 +1,73 @@ +nodes: + - id: camera + build: pip install -e ../../node-hub/dora-ios-lidar + path: dora-ios-lidar + _unstable_deploy: + machine: encoder-ios + inputs: + tick: dora/timer/millis/20 + outputs: + - image + - depth + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: rav1e-local + path: dora-rav1e + build: cargo build -p dora-rav1e --release + _unstable_deploy: + machine: encoder-ios + inputs: + image: camera/image + depth: camera/depth + outputs: + - image + - depth + env: + RAV1E_SPEED: 4 + + - id: dav1d-remote + path: dora-dav1d + build: cargo build -p dora-dav1d --release + _unstable_deploy: + machine: decoder + inputs: + image: rav1e-local/image + depth: rav1e-local/depth + outputs: + - image + - depth + + - id: rav1e-remote + path: dora-rav1e + build: cargo build -p dora-rav1e --release + _unstable_deploy: + machine: decoder + inputs: + image: dav1d-remote/image + depth: dav1d-remote/depth + outputs: + - image + - depth + + - id: dav1d-local + path: dora-dav1d + build: cargo build -p dora-dav1d --release + _unstable_deploy: + machine: encoder-ios + inputs: + image: rav1e-remote/image + depth: rav1e-remote/depth + outputs: + - image + - depth + + - id: plot + build: pip install -e ../../node-hub/dora-rerun + path: dora-rerun + _unstable_deploy: + machine: encoder-ios + inputs: + image: dav1d-remote/image + depth: dav1d-remote/depth diff --git a/examples/depth_camera/ios-dev.yaml b/examples/depth_camera/ios-dev.yaml index 076b2463..449e2845 100644 --- a/examples/depth_camera/ios-dev.yaml +++ b/examples/depth_camera/ios-dev.yaml @@ -7,6 +7,9 @@ nodes: outputs: - image - depth + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 - id: plot build: pip install -e ../../node-hub/dora-rerun diff --git a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py index 2615e61b..8c6a0b66 100644 --- a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py +++ b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py @@ -1,5 +1,6 @@ """TODO: Add docstring.""" +import os from threading import Event import cv2 @@ -9,6 +10,9 @@ from dora import Node from record3d import Record3DStream from scipy.spatial.transform import Rotation +image_width = os.getenv("IMAGE_WIDTH") +image_height = os.getenv("IMAGE_HEIGHT") + class DemoApp: """TODO: Add docstring.""" @@ -87,10 +91,23 @@ class DemoApp: intrinsic_mat = self.get_intrinsic_mat_from_coeffs( self.session.get_intrinsic_mat(), ) - pose = self.get_camera_pose() + # pose = self.get_camera_pose() if depth.shape != rgb.shape: rgb = cv2.resize(rgb, (depth.shape[1], depth.shape[0])) + if image_width is not None and image_height is not None: + f_0 = intrinsic_mat[0, 0] * (int(image_height) / rgb.shape[0]) + f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1]) + r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0]) + r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1]) + rgb = cv2.resize(rgb, (int(image_width), int(image_height))) + depth = cv2.resize(depth, (int(image_width), int(image_height))) + else: + f_0 = intrinsic_mat[0, 0] + f_1 = intrinsic_mat[1, 1] + r_0 = intrinsic_mat[0, 2] + r_1 = intrinsic_mat[1, 2] + node.send_output( "image", pa.array(rgb.ravel()), @@ -102,7 +119,7 @@ class DemoApp: ) depth = (np.array(depth) * 1_000).astype(np.uint16) - + depth = np.clip(depth, 0, 4095) # Clip depth to uint12 node.send_output( "depth", pa.array(depth.ravel()), @@ -111,16 +128,16 @@ class DemoApp: "height": depth.shape[0], "encoding": "mono16", "focal": [ - int(intrinsic_mat[0, 0]), - int(intrinsic_mat[1, 1]), + int(f_0), + int(f_1), ], "resolution": [ - int(intrinsic_mat[0, 2]), - int(intrinsic_mat[1, 2]), + int(r_0), + int(r_1), ], - "roll": pose[3], - "pitch": pose[4], - "yaw": pose[5], + # "roll": pose[3], + # "pitch": pose[4], # Adding 90 degrees to pitch + # "yaw": pose[5], }, ) diff --git a/node-hub/dora-ios-lidar/pyproject.toml b/node-hub/dora-ios-lidar/pyproject.toml index 3fe86c10..fb5a790d 100644 --- a/node-hub/dora-ios-lidar/pyproject.toml +++ b/node-hub/dora-ios-lidar/pyproject.toml @@ -5,9 +5,9 @@ authors = [{ name = "Your Name", email = "email@email.com" }] description = "dora-ios-lidar" license = { text = "MIT" } readme = "README.md" -requires-python = ">=3.8" +requires-python = "==3.8" -dependencies = ["dora-rs >= 0.3.9", "opencv-python>=4.11.0.86", "record3d>=1.4"] +dependencies = ["dora-rs >= 0.3.9", "opencv-python>=4.11.0.86", "record3d>=1.4", "scipy"] [dependency-groups] dev = ["pytest >=8.1.1", "ruff >=0.9.1"] diff --git a/node-hub/dora-rerun/src/lib.rs b/node-hub/dora-rerun/src/lib.rs index c482e718..c6884630 100644 --- a/node-hub/dora-rerun/src/lib.rs +++ b/node-hub/dora-rerun/src/lib.rs @@ -189,8 +189,8 @@ pub fn lib_main() -> Result<()> { } else { camera_pitch }; - let cos_theta = pitch.cos(); // np.cos(np.deg2rad(180-38)) - let sin_theta = pitch.sin(); // np.sin(np.deg2rad(180-38)) + let cos_theta = pitch.cos(); + let sin_theta = pitch.sin(); let points = match data.data_type() { dora_node_api::arrow::datatypes::DataType::Float64 => { From 75438baf730f5b760da7c6669ec384f190406212 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 16:42:49 +0200 Subject: [PATCH 18/26] Allow multiple version of python --- node-hub/dora-ios-lidar/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-hub/dora-ios-lidar/pyproject.toml b/node-hub/dora-ios-lidar/pyproject.toml index fb5a790d..0d6ed453 100644 --- a/node-hub/dora-ios-lidar/pyproject.toml +++ b/node-hub/dora-ios-lidar/pyproject.toml @@ -5,7 +5,7 @@ authors = [{ name = "Your Name", email = "email@email.com" }] description = "dora-ios-lidar" license = { text = "MIT" } readme = "README.md" -requires-python = "==3.8" +requires-python = ">=3.8" dependencies = ["dora-rs >= 0.3.9", "opencv-python>=4.11.0.86", "record3d>=1.4", "scipy"] From 9cc73848b5ac37558ad867a2fb46013b9206fffb Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 17:02:53 +0200 Subject: [PATCH 19/26] Update CI Apt version to get latest version of libdav1d --- .github/workflows/node-hub-ci-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml index 68b141ab..29902b7d 100644 --- a/.github/workflows/node-hub-ci-cd.yml +++ b/.github/workflows/node-hub-ci-cd.yml @@ -52,6 +52,7 @@ jobs: - name: Install system-level dependencies if: runner.os == 'Linux' run: | + sudo apt update sudo apt-get install portaudio19-dev sudo apt-get install libdav1d-dev nasm # Install mingw-w64 cross-compilers From bc51ab5ffa08a4b85fcd9df2e4a0c6cff5250e24 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 17:10:00 +0200 Subject: [PATCH 20/26] Bump node hub ci/cd to ubuntu 24.04 --- .github/workflows/node-hub-ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml index 29902b7d..e1c40de1 100644 --- a/.github/workflows/node-hub-ci-cd.yml +++ b/.github/workflows/node-hub-ci-cd.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: ${{ github.event_name != 'workflow_dispatch' && !(github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) }} matrix: - platform: [ubuntu-22.04, macos-14] + platform: [ubuntu-24.04, macos-14] folder: ${{ fromJson(needs.find-jobs.outputs.folders )}} steps: - name: Checkout repository From b143fd8eb3721051bb2bb1b80545d12a823429cd Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 17:20:19 +0200 Subject: [PATCH 21/26] Adding sacremoses into opus --- node-hub/dora-opus/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-hub/dora-opus/pyproject.toml b/node-hub/dora-opus/pyproject.toml index 344df864..30252328 100644 --- a/node-hub/dora-opus/pyproject.toml +++ b/node-hub/dora-opus/pyproject.toml @@ -15,11 +15,11 @@ requires-python = ">=3.8" dependencies = [ "dora-rs >= 0.3.9", "numpy < 2.0.0", - "transformers >= 4.45", "modelscope >= 1.18.1", "sentencepiece >= 0.1.99", "torch >= 2.2.0", + "sacremoses>=0.1.1", ] [dependency-groups] From 471327b29add8f70adf30d9c9f63ffa72c96f06d Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 17:54:26 +0200 Subject: [PATCH 22/26] Fix error when zenoh port is already taken --- binaries/daemon/src/lib.rs | 62 +++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index d71fe152..64115001 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -265,17 +265,11 @@ impl Daemon { let mut zenoh_config = zenoh::Config::default(); if let Some(addr) = coordinator_addr { - zenoh_config - .insert_json5( - "listen/endpoints", - r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#, - ) - .unwrap(); - // Linkstate make it possible to connect two daemons on different network through a public daemon zenoh_config .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) .unwrap(); + zenoh_config .insert_json5( "connect/endpoints", @@ -285,7 +279,40 @@ impl Daemon { ), ) .unwrap(); - if !addr.ip().is_loopback() { + zenoh_config + .insert_json5( + "listen/endpoints", + r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#, + ) + .unwrap(); + if cfg!(not(target_os = "linux")) { + warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); + zenoh_config + .insert_json5("scouting/multicast", r#"{ enabled: false }"#) + .unwrap(); + } + }; + if let Ok(zenoh_session) = zenoh::open(zenoh_config).await { + zenoh_session + } else { + warn!( + "failed to open zenoh session, retrying with default config + coordinator" + ); + let mut zenoh_config = zenoh::Config::default(); + zenoh_config + .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) + .unwrap(); + + if let Some(addr) = coordinator_addr { + zenoh_config + .insert_json5( + "connect/endpoints", + &format!( + r#"{{ router: ["tcp/[::]:7447"], peer: ["tcp/{}:5456"] }}"#, + addr.ip() + ), + ) + .unwrap(); if cfg!(not(target_os = "linux")) { warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); zenoh_config @@ -293,15 +320,16 @@ impl Daemon { .unwrap(); } } - }; - if let Ok(zenoh_session) = zenoh::open(zenoh_config).await { - zenoh_session - } else { - warn!("failed to open zenoh session, retrying with default config"); - zenoh::open(zenoh::Config::default()) - .await - .map_err(|e| eyre!(e)) - .wrap_err("failed to open zenoh session")? + if let Ok(zenoh_session) = zenoh::open(zenoh_config).await { + zenoh_session + } else { + warn!("failed to open zenoh session, retrying with default config"); + let zenoh_config = zenoh::Config::default(); + zenoh::open(zenoh_config) + .await + .map_err(|e| eyre!(e)) + .context("failed to open zenoh session")? + } } } Err(std::env::VarError::NotUnicode(_)) => eyre::bail!( From 714459f6a608ed0f193864eafd7f13803bd8c185 Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 18:08:25 +0200 Subject: [PATCH 23/26] Remove test for opus in favor of dora-phi4 --- node-hub/dora-opus/tests/test_translate.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/node-hub/dora-opus/tests/test_translate.py b/node-hub/dora-opus/tests/test_translate.py index 34db2f82..f03028d0 100644 --- a/node-hub/dora-opus/tests/test_translate.py +++ b/node-hub/dora-opus/tests/test_translate.py @@ -1,12 +1,11 @@ """TODO: Add docstring.""" -import pytest - def test_import_main(): """TODO: Add docstring.""" - from dora_opus.main import main + pass # OPUS is no longer maintained in favor of dora-phi4 + # from dora_opus.main import main # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow. - with pytest.raises(RuntimeError): - main() + # nwith pytest.raises(RuntimeError): + # nmain() From b2b849c21ab35d68f39ebf5fc7b3ba07823b931c Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 18:32:55 +0200 Subject: [PATCH 24/26] Depreciated outetts for dora-kokorotts --- node-hub/dora-outtetts/README.md | 2 ++ .../dora_outtetts/tests/test_main.py | 18 ++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/node-hub/dora-outtetts/README.md b/node-hub/dora-outtetts/README.md index 6c015b4f..8e0058a4 100644 --- a/node-hub/dora-outtetts/README.md +++ b/node-hub/dora-outtetts/README.md @@ -1,5 +1,7 @@ # dora-outtetts +> dora-outtetts is no longer maintained in favor of dora-kokorotts + ## Getting started - Install it with pip: diff --git a/node-hub/dora-outtetts/dora_outtetts/tests/test_main.py b/node-hub/dora-outtetts/dora_outtetts/tests/test_main.py index ca1c6dd0..943883f0 100644 --- a/node-hub/dora-outtetts/dora_outtetts/tests/test_main.py +++ b/node-hub/dora-outtetts/dora_outtetts/tests/test_main.py @@ -2,26 +2,16 @@ import os -import pytest - -from dora_outtetts.main import load_interface, main - CI = os.getenv("CI", "false") in ["True", "true"] def test_import_main(): """TODO: Add docstring.""" - with pytest.raises(RuntimeError): - main([]) + pass # Outetts is no longer maintained in favor of dora-kokorotts + # with pytest.raises(RuntimeError): + # main([]) def test_load_interface(): """TODO: Add docstring.""" - try: - interface = load_interface() - except RuntimeError: - # Error raised by MPS out of memory. - if CI: - interface = "ok" - - assert interface is not None + pass From 98c2227375a76deb92e21b32ccc2fcc6e15588af Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 18:51:35 +0200 Subject: [PATCH 25/26] Fix windows example that is failing due to linkstate --- binaries/daemon/src/lib.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 64115001..0a8a13d8 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -266,9 +266,12 @@ impl Daemon { if let Some(addr) = coordinator_addr { // Linkstate make it possible to connect two daemons on different network through a public daemon - zenoh_config - .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) - .unwrap(); + // TODO: There is currently a CI/CD Error in windows linkstate. + if cfg!(not(target_os = "windows")) { + zenoh_config + .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) + .unwrap(); + } zenoh_config .insert_json5( @@ -299,9 +302,13 @@ impl Daemon { "failed to open zenoh session, retrying with default config + coordinator" ); let mut zenoh_config = zenoh::Config::default(); - zenoh_config - .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) - .unwrap(); + // Linkstate make it possible to connect two daemons on different network through a public daemon + // TODO: There is currently a CI/CD Error in windows linkstate. + if cfg!(not(target_os = "windows")) { + zenoh_config + .insert_json5("routing/peer", r#"{ mode: "linkstate" }"#) + .unwrap(); + } if let Some(addr) = coordinator_addr { zenoh_config From 1d5b917c0ff0ccbeb28c6cd27039620ae510f0af Mon Sep 17 00:00:00 2001 From: haixuantao Date: Sun, 6 Apr 2025 19:18:53 +0200 Subject: [PATCH 26/26] try multicast on windows --- binaries/daemon/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 0a8a13d8..a204ad02 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -288,8 +288,8 @@ impl Daemon { r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#, ) .unwrap(); - if cfg!(not(target_os = "linux")) { - warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); + if cfg!(target_os = "macos") { + warn!("disabling multicast on macos systems. Enable it with the ZENOH_CONFIG env variable or file"); zenoh_config .insert_json5("scouting/multicast", r#"{ enabled: false }"#) .unwrap(); @@ -320,8 +320,8 @@ impl Daemon { ), ) .unwrap(); - if cfg!(not(target_os = "linux")) { - warn!("disabling multicast on non-linux systems. Enable it with the ZENOH_CONFIG env variable or file"); + if cfg!(target_os = "macos") { + warn!("disabling multicast on macos systems. Enable it with the ZENOH_CONFIG env variable or file"); zenoh_config .insert_json5("scouting/multicast", r#"{ enabled: false }"#) .unwrap();