dora-rs
/
dora
Not watched
1
0
Fork 0
Code
Releases 124 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Minor fix on rav1e dav1d

tags/v0.3.12-rc0
haixuantao haixuanTao 9 months ago
parent
70074749ba
commit
48f72dce5a
8 changed files with 168 additions and 64 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +13
    -3
      Cargo.lock
  2. +0
    -7
      examples/av1-encoding/dataflow.yml
  3. +17
    -36
      examples/av1-encoding/ios-dev.yaml
  4. +91
    -14
      node-hub/dora-dav1d/src/lib.rs
  5. +9
    -1
      node-hub/dora-ios-lidar/dora_ios_lidar/main.py
  6. +14
    -1
      node-hub/dora-rav1e/src/lib.rs
  7. +17
    -1
      node-hub/opencv-plot/opencv_plot/main.py
  8. +7
    -1
      node-hub/opencv-plot/pyproject.toml

+ 13
- 3
Cargo.lock View File

@@ -1350,6 +1350,15 @@ dependencies = [
"v_frame", "v_frame",
] ]


[[package]]
name = "avif-serialize"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e"
dependencies = [
"arrayvec",
]

[[package]] [[package]]
name = "axum" name = "axum"
version = "0.7.9" version = "0.7.9"
@@ -3438,6 +3447,7 @@ dependencies = [
name = "dora-rav1e" name = "dora-rav1e"
version = "0.3.11+fix1" version = "0.3.11+fix1"
dependencies = [ dependencies = [
"avif-serialize",
"bytemuck", "bytemuck",
"dora-node-api", "dora-node-api",
"eyre", "eyre",
@@ -5405,7 +5415,7 @@ dependencies = [
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project-lite", "pin-project-lite",
"socket2 0.4.10",
"socket2 0.5.8",
"tokio", "tokio",
"tower-service", "tower-service",
"tracing", "tracing",
@@ -6368,7 +6378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
] ]


[[package]] [[package]]
@@ -13923,7 +13933,7 @@ version = "1.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
dependencies = [ dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"static_assertions", "static_assertions",
] ]




+ 0
- 7
examples/av1-encoding/dataflow.yml View File

@@ -42,9 +42,6 @@ nodes:
image: dav1d-remote/image image: dav1d-remote/image
outputs: outputs:
- image - image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480


- id: dav1d-local - id: dav1d-local
path: dora-dav1d path: dora-dav1d
@@ -55,9 +52,6 @@ nodes:
image: rav1e-remote/image image: rav1e-remote/image
outputs: outputs:
- image - image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480


- id: plot - id: plot
build: pip install -e ../../node-hub/dora-rerun build: pip install -e ../../node-hub/dora-rerun
@@ -66,4 +60,3 @@ nodes:
path: dora-rerun path: dora-rerun
inputs: inputs:
image_decode: dav1d-local/image image_decode: dav1d-local/image
image_echo: echo/image

+ 17
- 36
examples/av1-encoding/ios-dev.yaml View File

@@ -2,72 +2,53 @@ nodes:
- id: camera - id: camera
build: pip install -e ../../node-hub/dora-ios-lidar build: pip install -e ../../node-hub/dora-ios-lidar
path: dora-ios-lidar path: dora-ios-lidar
_unstable_deploy:
machine: encoder-ios
inputs: inputs:
tick: dora/timer/millis/20 tick: dora/timer/millis/20
outputs: outputs:
- image - image
- depth - depth
env: env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480
IMAGE_WIDTH: 1280
IMAGE_HEIGHT: 720
ROTATE: ROTATE_90_CLOCKWISE


- id: rav1e-local - id: rav1e-local
path: dora-rav1e path: dora-rav1e
build: cargo build -p dora-rav1e --release build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder-ios
inputs: inputs:
image: camera/image image: camera/image
depth: camera/depth
outputs: outputs:
- image - image
- depth
env: env:
RAV1E_SPEED: 4
RAV1E_SPEED: 10


- id: dav1d-remote
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: decoder
- id: rav1e-local-depth
path: dora-rav1e
build: cargo build -p dora-rav1e --release
inputs: inputs:
image: rav1e-local/image
depth: rav1e-local/depth
depth: camera/depth
outputs: outputs:
- image
- depth - depth

- id: rav1e-remote
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: decoder
env:
RAV1E_SPEED: 10
- id: dav1d-local-depth
path: dora-dav1d
build: cargo build -p dora-dav1d --release
inputs: inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth
depth: rav1e-local-depth/depth
outputs: outputs:
- image
- depth - depth

- id: dav1d-local - id: dav1d-local
path: dora-dav1d path: dora-dav1d
build: cargo build -p dora-dav1d --release build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: encoder-ios
inputs: inputs:
image: rav1e-remote/image
depth: rav1e-remote/depth
image: rav1e-local/image
outputs: outputs:
- image - image
- depth


- id: plot - id: plot
build: pip install -e ../../node-hub/dora-rerun build: pip install -e ../../node-hub/dora-rerun
path: dora-rerun path: dora-rerun
_unstable_deploy:
machine: encoder-ios
inputs: inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth
image: dav1d-local/image
depth: dav1d-local-depth/depth

+ 91
- 14
node-hub/dora-dav1d/src/lib.rs View File

@@ -1,3 +1,5 @@
use std::env::var;

use dav1d::Settings; use dav1d::Settings;
use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow}; use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow};
use eyre::{Context, Result}; use eyre::{Context, Result};
@@ -49,6 +51,8 @@ pub fn lib_main() -> Result<()> {
let (mut node, mut events) = let (mut node, mut events) =
DoraNode::init_from_env().context("Could not initialize dora node")?; DoraNode::init_from_env().context("Could not initialize dora node")?;


let output_encoding = var("ENCODING").unwrap_or("bgr8".to_string());

loop { loop {
match events.recv() { match events.recv() {
Some(Event::Input { Some(Event::Input {
@@ -82,23 +86,96 @@ pub fn lib_main() -> Result<()> {
let y = p.plane(dav1d::PlanarImageComponent::Y); let y = p.plane(dav1d::PlanarImageComponent::Y);
let u = p.plane(dav1d::PlanarImageComponent::U); let u = p.plane(dav1d::PlanarImageComponent::U);
let v = p.plane(dav1d::PlanarImageComponent::V); let v = p.plane(dav1d::PlanarImageComponent::V);
let y = yuv420_to_bgr(&y, &u, &v, p.width(), p.height());
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("bgr8".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
match output_encoding.as_str() {
"yuv420" => {
let mut y = y.to_vec();
let mut u = u.to_vec();
let mut v = v.to_vec();
y.append(&mut u);
y.append(&mut v);
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"yuv420".to_string(),
),
);
metadata.parameters.insert(
"width".to_string(),
dora_node_api::Parameter::Integer(
p.width() as i64
),
);
metadata.parameters.insert(
"height".to_string(),
dora_node_api::Parameter::Integer(
p.height() as i64
),
);

node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
"bgr8" => {
let y = yuv420_to_bgr(
&y,
&u,
&v,
p.width(),
p.height(),
);
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"bgr8".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
_ => {
warn!(
"Unsupported output encoding {}",
output_encoding
);
continue;
}
}
} }
dav1d::PixelLayout::I400 => { dav1d::PixelLayout::I400 => {
let y = p.plane(dav1d::PlanarImageComponent::Y); let y = p.plane(dav1d::PlanarImageComponent::Y);
let vec16: Vec<u16> = bytemuck::cast_slice(&y).to_vec();
let arrow = vec16.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("mono16".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
match p.bit_depth() {
8 => {
let y = y.to_vec();
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"mono8".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
10 | 12 => {
let vec16: Vec<u16> =
bytemuck::cast_slice(&y).to_vec();
let arrow = vec16.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"mono16".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
_ => {
warn!("Unsupported bit depth {}", p.bit_depth());
continue;
}
}
} }
_ => { _ => {
warn!("Unsupported pixel layout"); warn!("Unsupported pixel layout");


+ 9
- 1
node-hub/dora-ios-lidar/dora_ios_lidar/main.py View File

@@ -12,6 +12,7 @@ from scipy.spatial.transform import Rotation


image_width = os.getenv("IMAGE_WIDTH") image_width = os.getenv("IMAGE_WIDTH")
image_height = os.getenv("IMAGE_HEIGHT") image_height = os.getenv("IMAGE_HEIGHT")
ROTATE = os.getenv("ROTATE")




class DemoApp: class DemoApp:
@@ -100,8 +101,15 @@ class DemoApp:
f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1]) f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1])
r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0]) r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0])
r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1]) r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1])
if ROTATE == "ROTATE_90_CLOCKWISE":
rgb = cv2.rotate(rgb, cv2.ROTATE_90_CLOCKWISE)
depth = cv2.rotate(depth, cv2.ROTATE_90_CLOCKWISE)
rgb = cv2.resize(rgb, (int(image_width), int(image_height))) rgb = cv2.resize(rgb, (int(image_width), int(image_height)))
depth = cv2.resize(depth, (int(image_width), int(image_height)))
depth = cv2.resize(
depth,
(int(image_width), int(image_height)),
interpolation=cv2.INTER_NEAREST,
)
else: else:
f_0 = intrinsic_mat[0, 0] f_0 = intrinsic_mat[0, 0]
f_1 = intrinsic_mat[1, 1] f_1 = intrinsic_mat[1, 1]


+ 14
- 1
node-hub/dora-rav1e/src/lib.rs View File

@@ -201,6 +201,13 @@ fn send_yuv(
metadata metadata
.parameters .parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string())); .insert("encoding".to_string(), Parameter::String("av1".to_string()));
metadata
.parameters
.insert("height".to_string(), Parameter::Integer(enc.height as i64));
metadata
.parameters
.insert("width".to_string(), Parameter::Integer(enc.width as i64));

let data = pkt.data; let data = pkt.data;
let arrow = data.into_arrow(); let arrow = data.into_arrow();
node.send_output(id, metadata.parameters.clone(), arrow) node.send_output(id, metadata.parameters.clone(), arrow)
@@ -274,6 +281,12 @@ pub fn lib_main() -> Result<()> {
height, height,
speed_settings: SpeedSettings::from_preset(speed), speed_settings: SpeedSettings::from_preset(speed),
low_latency: true, low_latency: true,
chroma_sampling: color::ChromaSampling::Cs420,
color_description: Some(ColorDescription {
matrix_coefficients: MatrixCoefficients::BT709,
transfer_characteristics: color::TransferCharacteristics::BT709,
color_primaries: color::ColorPrimaries::BT709,
}),
..Default::default() ..Default::default()
}; };
match encoding { match encoding {
@@ -327,9 +340,9 @@ pub fn lib_main() -> Result<()> {
fill_zeros_toward_center_y_plane_in_place(&mut buffer, width, height); fill_zeros_toward_center_y_plane_in_place(&mut buffer, width, height);
} }


// let buffer = shift_u16_slice_to_upper_12_bits(buffer);
let bytes: &[u8] = &bytemuck::cast_slice(&buffer); let bytes: &[u8] = &bytemuck::cast_slice(&buffer);


let cfg = Config::new().with_encoder_config(enc.clone());
let mut ctx: Context<u16> = cfg.new_context().unwrap(); let mut ctx: Context<u16> = cfg.new_context().unwrap();
let mut f = ctx.new_frame(); let mut f = ctx.new_frame();




+ 17
- 1
node-hub/opencv-plot/opencv_plot/main.py View File

@@ -1,12 +1,19 @@
"""TODO: Add docstring.""" """TODO: Add docstring."""


import argparse import argparse
import io
import os import os


import cv2 import cv2
import numpy as np import numpy as np
import pyarrow as pa import pyarrow as pa
from dora import Node from dora import Node
from PIL import (
Image,
)

if True:
import pillow_avif # noqa # noqa


RUNNER_CI = True if os.getenv("CI") == "true" else False RUNNER_CI = True if os.getenv("CI") == "true" else False


@@ -76,6 +83,7 @@ def plot_frame(plot):


def yuv420p_to_bgr_opencv(yuv_array, width, height): def yuv420p_to_bgr_opencv(yuv_array, width, height):
"""TODO: Add docstring.""" """TODO: Add docstring."""
yuv_array = yuv_array[: width * height * 3 // 2]
yuv = yuv_array.reshape((height * 3 // 2, width)) yuv = yuv_array.reshape((height * 3 // 2, width))
return cv2.cvtColor(yuv, cv2.COLOR_YUV420p2RGB) return cv2.cvtColor(yuv, cv2.COLOR_YUV420p2RGB)


@@ -145,7 +153,6 @@ def main():
encoding = metadata["encoding"] encoding = metadata["encoding"]
width = metadata["width"] width = metadata["width"]
height = metadata["height"] height = metadata["height"]

if encoding == "bgr8": if encoding == "bgr8":
channels = 3 channels = 3
storage_type = np.uint8 storage_type = np.uint8
@@ -181,6 +188,14 @@ def main():
img_bgr_restored = yuv420p_to_bgr_opencv(storage, width, height) img_bgr_restored = yuv420p_to_bgr_opencv(storage, width, height)


plot.frame = img_bgr_restored plot.frame = img_bgr_restored
elif encoding == "avif":
# Convert AVIF to RGB
array = storage.to_numpy()
bytes = array.tobytes()
img = Image.open(io.BytesIO(bytes))
img = img.convert("RGB")
plot.frame = np.array(img)
plot.frame = cv2.cvtColor(plot.frame, cv2.COLOR_RGB2BGR)
else: else:
raise RuntimeError(f"Unsupported image encoding: {encoding}") raise RuntimeError(f"Unsupported image encoding: {encoding}")


@@ -188,6 +203,7 @@ def main():
if not RUNNER_CI: if not RUNNER_CI:
if cv2.waitKey(1) & 0xFF == ord("q"): if cv2.waitKey(1) & 0xFF == ord("q"):
break break

elif event_id == "bbox": elif event_id == "bbox":
arrow_bbox = event["value"][0] arrow_bbox = event["value"][0]
bbox_format = event["metadata"]["format"] bbox_format = event["metadata"]["format"]


+ 7
- 1
node-hub/opencv-plot/pyproject.toml View File

@@ -10,7 +10,13 @@ description = "Dora Node for plotting text and bbox on image with OpenCV"


requires-python = ">=3.8" requires-python = ">=3.8"


dependencies = ["dora-rs >= 0.3.9", "numpy < 2.0.0", "opencv-python >= 4.1.1"]
dependencies = [
"dora-rs >= 0.3.9",
"numpy < 2.0.0",
"opencv-python >= 4.1.1",
"pillow-avif-plugin>=1.5.1",
"pillow>=10.4.0",
]


[dependency-groups] [dependency-groups]
dev = ["pytest >=8.1.1", "ruff >=0.9.1"] dev = ["pytest >=8.1.1", "ruff >=0.9.1"]


Write Preview
Loading…
Cancel
Save