diff --git a/Cargo.lock b/Cargo.lock index e62a3a37..29727563 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,6 +1350,15 @@ dependencies = [ "v_frame", ] +[[package]] +name = "avif-serialize" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e" +dependencies = [ + "arrayvec", +] + [[package]] name = "axum" version = "0.7.9" @@ -3438,6 +3447,7 @@ dependencies = [ name = "dora-rav1e" version = "0.3.11+fix1" dependencies = [ + "avif-serialize", "bytemuck", "dora-node-api", "eyre", @@ -5405,7 +5415,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -6368,7 +6378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if 1.0.0", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -13923,7 +13933,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] diff --git a/examples/av1-encoding/dataflow.yml b/examples/av1-encoding/dataflow.yml index 131da27a..2dbcef01 100644 --- a/examples/av1-encoding/dataflow.yml +++ b/examples/av1-encoding/dataflow.yml @@ -42,9 +42,6 @@ nodes: image: dav1d-remote/image outputs: - image - env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - id: dav1d-local path: dora-dav1d @@ -55,9 +52,6 @@ nodes: image: rav1e-remote/image outputs: - image - env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - id: plot build: pip install -e ../../node-hub/dora-rerun @@ -66,4 +60,3 @@ nodes: path: dora-rerun inputs: image_decode: dav1d-local/image - image_echo: echo/image diff --git a/examples/av1-encoding/ios-dev.yaml b/examples/av1-encoding/ios-dev.yaml index d14fdb36..02fdf900 100644 --- a/examples/av1-encoding/ios-dev.yaml +++ b/examples/av1-encoding/ios-dev.yaml @@ -2,72 +2,53 @@ nodes: - id: camera build: pip install -e ../../node-hub/dora-ios-lidar path: dora-ios-lidar - _unstable_deploy: - machine: encoder-ios inputs: tick: dora/timer/millis/20 outputs: - image - depth env: - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 + IMAGE_WIDTH: 1280 + IMAGE_HEIGHT: 720 + ROTATE: ROTATE_90_CLOCKWISE - id: rav1e-local path: dora-rav1e build: cargo build -p dora-rav1e --release - _unstable_deploy: - machine: encoder-ios inputs: image: camera/image - depth: camera/depth outputs: - image - - depth env: - RAV1E_SPEED: 4 + RAV1E_SPEED: 10 - - id: dav1d-remote - path: dora-dav1d - build: cargo build -p dora-dav1d --release - _unstable_deploy: - machine: decoder + - id: rav1e-local-depth + path: dora-rav1e + build: cargo build -p dora-rav1e --release inputs: - image: rav1e-local/image - depth: rav1e-local/depth + depth: camera/depth outputs: - - image - depth - - - id: rav1e-remote - path: dora-rav1e - build: cargo build -p dora-rav1e --release - _unstable_deploy: - machine: decoder + env: + RAV1E_SPEED: 10 + - id: dav1d-local-depth + path: dora-dav1d + build: cargo build -p dora-dav1d --release inputs: - image: dav1d-remote/image - depth: dav1d-remote/depth + depth: rav1e-local-depth/depth outputs: - - image - depth - - id: dav1d-local path: dora-dav1d build: cargo build -p dora-dav1d --release - _unstable_deploy: - machine: encoder-ios inputs: - image: rav1e-remote/image - depth: rav1e-remote/depth + image: rav1e-local/image outputs: - image - - depth - id: plot build: pip install -e ../../node-hub/dora-rerun path: dora-rerun - _unstable_deploy: - machine: encoder-ios inputs: - image: dav1d-remote/image - depth: dav1d-remote/depth + image: dav1d-local/image + depth: dav1d-local-depth/depth diff --git a/node-hub/dora-dav1d/src/lib.rs b/node-hub/dora-dav1d/src/lib.rs index a829f5aa..b6aeed30 100644 --- a/node-hub/dora-dav1d/src/lib.rs +++ b/node-hub/dora-dav1d/src/lib.rs @@ -1,3 +1,5 @@ +use std::env::var; + use dav1d::Settings; use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow}; use eyre::{Context, Result}; @@ -49,6 +51,8 @@ pub fn lib_main() -> Result<()> { let (mut node, mut events) = DoraNode::init_from_env().context("Could not initialize dora node")?; + let output_encoding = var("ENCODING").unwrap_or("bgr8".to_string()); + loop { match events.recv() { Some(Event::Input { @@ -82,23 +86,96 @@ pub fn lib_main() -> Result<()> { let y = p.plane(dav1d::PlanarImageComponent::Y); let u = p.plane(dav1d::PlanarImageComponent::U); let v = p.plane(dav1d::PlanarImageComponent::V); - let y = yuv420_to_bgr(&y, &u, &v, p.width(), p.height()); - let arrow = y.into_arrow(); - metadata.parameters.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("bgr8".to_string()), - ); - node.send_output(id, metadata.parameters, arrow).unwrap(); + match output_encoding.as_str() { + "yuv420" => { + let mut y = y.to_vec(); + let mut u = u.to_vec(); + let mut v = v.to_vec(); + y.append(&mut u); + y.append(&mut v); + let arrow = y.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String( + "yuv420".to_string(), + ), + ); + metadata.parameters.insert( + "width".to_string(), + dora_node_api::Parameter::Integer( + p.width() as i64 + ), + ); + metadata.parameters.insert( + "height".to_string(), + dora_node_api::Parameter::Integer( + p.height() as i64 + ), + ); + + node.send_output(id, metadata.parameters, arrow) + .unwrap(); + } + "bgr8" => { + let y = yuv420_to_bgr( + &y, + &u, + &v, + p.width(), + p.height(), + ); + let arrow = y.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String( + "bgr8".to_string(), + ), + ); + node.send_output(id, metadata.parameters, arrow) + .unwrap(); + } + _ => { + warn!( + "Unsupported output encoding {}", + output_encoding + ); + continue; + } + } } dav1d::PixelLayout::I400 => { let y = p.plane(dav1d::PlanarImageComponent::Y); - let vec16: Vec = bytemuck::cast_slice(&y).to_vec(); - let arrow = vec16.into_arrow(); - metadata.parameters.insert( - "encoding".to_string(), - dora_node_api::Parameter::String("mono16".to_string()), - ); - node.send_output(id, metadata.parameters, arrow).unwrap(); + match p.bit_depth() { + 8 => { + let y = y.to_vec(); + let arrow = y.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String( + "mono8".to_string(), + ), + ); + node.send_output(id, metadata.parameters, arrow) + .unwrap(); + } + 10 | 12 => { + let vec16: Vec = + bytemuck::cast_slice(&y).to_vec(); + let arrow = vec16.into_arrow(); + metadata.parameters.insert( + "encoding".to_string(), + dora_node_api::Parameter::String( + "mono16".to_string(), + ), + ); + node.send_output(id, metadata.parameters, arrow) + .unwrap(); + } + _ => { + warn!("Unsupported bit depth {}", p.bit_depth()); + continue; + } + } } _ => { warn!("Unsupported pixel layout"); diff --git a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py index 8c6a0b66..13c67524 100644 --- a/node-hub/dora-ios-lidar/dora_ios_lidar/main.py +++ b/node-hub/dora-ios-lidar/dora_ios_lidar/main.py @@ -12,6 +12,7 @@ from scipy.spatial.transform import Rotation image_width = os.getenv("IMAGE_WIDTH") image_height = os.getenv("IMAGE_HEIGHT") +ROTATE = os.getenv("ROTATE") class DemoApp: @@ -100,8 +101,15 @@ class DemoApp: f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1]) r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0]) r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1]) + if ROTATE == "ROTATE_90_CLOCKWISE": + rgb = cv2.rotate(rgb, cv2.ROTATE_90_CLOCKWISE) + depth = cv2.rotate(depth, cv2.ROTATE_90_CLOCKWISE) rgb = cv2.resize(rgb, (int(image_width), int(image_height))) - depth = cv2.resize(depth, (int(image_width), int(image_height))) + depth = cv2.resize( + depth, + (int(image_width), int(image_height)), + interpolation=cv2.INTER_NEAREST, + ) else: f_0 = intrinsic_mat[0, 0] f_1 = intrinsic_mat[1, 1] diff --git a/node-hub/dora-rav1e/src/lib.rs b/node-hub/dora-rav1e/src/lib.rs index 6884f7b6..2491d42f 100644 --- a/node-hub/dora-rav1e/src/lib.rs +++ b/node-hub/dora-rav1e/src/lib.rs @@ -201,6 +201,13 @@ fn send_yuv( metadata .parameters .insert("encoding".to_string(), Parameter::String("av1".to_string())); + metadata + .parameters + .insert("height".to_string(), Parameter::Integer(enc.height as i64)); + metadata + .parameters + .insert("width".to_string(), Parameter::Integer(enc.width as i64)); + let data = pkt.data; let arrow = data.into_arrow(); node.send_output(id, metadata.parameters.clone(), arrow) @@ -274,6 +281,12 @@ pub fn lib_main() -> Result<()> { height, speed_settings: SpeedSettings::from_preset(speed), low_latency: true, + chroma_sampling: color::ChromaSampling::Cs420, + color_description: Some(ColorDescription { + matrix_coefficients: MatrixCoefficients::BT709, + transfer_characteristics: color::TransferCharacteristics::BT709, + color_primaries: color::ColorPrimaries::BT709, + }), ..Default::default() }; match encoding { @@ -327,9 +340,9 @@ pub fn lib_main() -> Result<()> { fill_zeros_toward_center_y_plane_in_place(&mut buffer, width, height); } - // let buffer = shift_u16_slice_to_upper_12_bits(buffer); let bytes: &[u8] = &bytemuck::cast_slice(&buffer); + let cfg = Config::new().with_encoder_config(enc.clone()); let mut ctx: Context = cfg.new_context().unwrap(); let mut f = ctx.new_frame(); diff --git a/node-hub/opencv-plot/opencv_plot/main.py b/node-hub/opencv-plot/opencv_plot/main.py index a7007d42..78b26756 100644 --- a/node-hub/opencv-plot/opencv_plot/main.py +++ b/node-hub/opencv-plot/opencv_plot/main.py @@ -1,12 +1,19 @@ """TODO: Add docstring.""" import argparse +import io import os import cv2 import numpy as np import pyarrow as pa from dora import Node +from PIL import ( + Image, +) + +if True: + import pillow_avif # noqa # noqa RUNNER_CI = True if os.getenv("CI") == "true" else False @@ -76,6 +83,7 @@ def plot_frame(plot): def yuv420p_to_bgr_opencv(yuv_array, width, height): """TODO: Add docstring.""" + yuv_array = yuv_array[: width * height * 3 // 2] yuv = yuv_array.reshape((height * 3 // 2, width)) return cv2.cvtColor(yuv, cv2.COLOR_YUV420p2RGB) @@ -145,7 +153,6 @@ def main(): encoding = metadata["encoding"] width = metadata["width"] height = metadata["height"] - if encoding == "bgr8": channels = 3 storage_type = np.uint8 @@ -181,6 +188,14 @@ def main(): img_bgr_restored = yuv420p_to_bgr_opencv(storage, width, height) plot.frame = img_bgr_restored + elif encoding == "avif": + # Convert AVIF to RGB + array = storage.to_numpy() + bytes = array.tobytes() + img = Image.open(io.BytesIO(bytes)) + img = img.convert("RGB") + plot.frame = np.array(img) + plot.frame = cv2.cvtColor(plot.frame, cv2.COLOR_RGB2BGR) else: raise RuntimeError(f"Unsupported image encoding: {encoding}") @@ -188,6 +203,7 @@ def main(): if not RUNNER_CI: if cv2.waitKey(1) & 0xFF == ord("q"): break + elif event_id == "bbox": arrow_bbox = event["value"][0] bbox_format = event["metadata"]["format"] diff --git a/node-hub/opencv-plot/pyproject.toml b/node-hub/opencv-plot/pyproject.toml index 95f253ee..6b4412a4 100644 --- a/node-hub/opencv-plot/pyproject.toml +++ b/node-hub/opencv-plot/pyproject.toml @@ -10,7 +10,13 @@ description = "Dora Node for plotting text and bbox on image with OpenCV" requires-python = ">=3.8" -dependencies = ["dora-rs >= 0.3.9", "numpy < 2.0.0", "opencv-python >= 4.1.1"] +dependencies = [ + "dora-rs >= 0.3.9", + "numpy < 2.0.0", + "opencv-python >= 4.1.1", + "pillow-avif-plugin>=1.5.1", + "pillow>=10.4.0", +] [dependency-groups] dev = ["pytest >=8.1.1", "ruff >=0.9.1"]