dora-rs
/
dora
Not watched
1
0
Fork 0
Code
Releases 124 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Minor fix on rav1e dav1d

tags/v0.3.12-rc0
haixuantao haixuanTao 9 months ago
parent
70074749ba
commit
48f72dce5a
8 changed files with 168 additions and 64 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +13
    -3
      Cargo.lock
  2. +0
    -7
      examples/av1-encoding/dataflow.yml
  3. +17
    -36
      examples/av1-encoding/ios-dev.yaml
  4. +91
    -14
      node-hub/dora-dav1d/src/lib.rs
  5. +9
    -1
      node-hub/dora-ios-lidar/dora_ios_lidar/main.py
  6. +14
    -1
      node-hub/dora-rav1e/src/lib.rs
  7. +17
    -1
      node-hub/opencv-plot/opencv_plot/main.py
  8. +7
    -1
      node-hub/opencv-plot/pyproject.toml

+ 13
- 3
Cargo.lock View File

@@ -1350,6 +1350,15 @@ dependencies = [
"v_frame",
]

[[package]]
name = "avif-serialize"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e"
dependencies = [
"arrayvec",
]

[[package]]
name = "axum"
version = "0.7.9"
@@ -3438,6 +3447,7 @@ dependencies = [
name = "dora-rav1e"
version = "0.3.11+fix1"
dependencies = [
"avif-serialize",
"bytemuck",
"dora-node-api",
"eyre",
@@ -5405,7 +5415,7 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.4.10",
"socket2 0.5.8",
"tokio",
"tower-service",
"tracing",
@@ -6368,7 +6378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
dependencies = [
"cfg-if 1.0.0",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
]

[[package]]
@@ -13923,7 +13933,7 @@ version = "1.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"static_assertions",
]



+ 0
- 7
examples/av1-encoding/dataflow.yml View File

@@ -42,9 +42,6 @@ nodes:
image: dav1d-remote/image
outputs:
- image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: dav1d-local
path: dora-dav1d
@@ -55,9 +52,6 @@ nodes:
image: rav1e-remote/image
outputs:
- image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: plot
build: pip install -e ../../node-hub/dora-rerun
@@ -66,4 +60,3 @@ nodes:
path: dora-rerun
inputs:
image_decode: dav1d-local/image
image_echo: echo/image

+ 17
- 36
examples/av1-encoding/ios-dev.yaml View File

@@ -2,72 +2,53 @@ nodes:
- id: camera
build: pip install -e ../../node-hub/dora-ios-lidar
path: dora-ios-lidar
_unstable_deploy:
machine: encoder-ios
inputs:
tick: dora/timer/millis/20
outputs:
- image
- depth
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480
IMAGE_WIDTH: 1280
IMAGE_HEIGHT: 720
ROTATE: ROTATE_90_CLOCKWISE

- id: rav1e-local
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder-ios
inputs:
image: camera/image
depth: camera/depth
outputs:
- image
- depth
env:
RAV1E_SPEED: 4
RAV1E_SPEED: 10

- id: dav1d-remote
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: decoder
- id: rav1e-local-depth
path: dora-rav1e
build: cargo build -p dora-rav1e --release
inputs:
image: rav1e-local/image
depth: rav1e-local/depth
depth: camera/depth
outputs:
- image
- depth

- id: rav1e-remote
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: decoder
env:
RAV1E_SPEED: 10
- id: dav1d-local-depth
path: dora-dav1d
build: cargo build -p dora-dav1d --release
inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth
depth: rav1e-local-depth/depth
outputs:
- image
- depth

- id: dav1d-local
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: encoder-ios
inputs:
image: rav1e-remote/image
depth: rav1e-remote/depth
image: rav1e-local/image
outputs:
- image
- depth

- id: plot
build: pip install -e ../../node-hub/dora-rerun
path: dora-rerun
_unstable_deploy:
machine: encoder-ios
inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth
image: dav1d-local/image
depth: dav1d-local-depth/depth

+ 91
- 14
node-hub/dora-dav1d/src/lib.rs View File

@@ -1,3 +1,5 @@
use std::env::var;

use dav1d::Settings;
use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow};
use eyre::{Context, Result};
@@ -49,6 +51,8 @@ pub fn lib_main() -> Result<()> {
let (mut node, mut events) =
DoraNode::init_from_env().context("Could not initialize dora node")?;

let output_encoding = var("ENCODING").unwrap_or("bgr8".to_string());

loop {
match events.recv() {
Some(Event::Input {
@@ -82,23 +86,96 @@ pub fn lib_main() -> Result<()> {
let y = p.plane(dav1d::PlanarImageComponent::Y);
let u = p.plane(dav1d::PlanarImageComponent::U);
let v = p.plane(dav1d::PlanarImageComponent::V);
let y = yuv420_to_bgr(&y, &u, &v, p.width(), p.height());
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("bgr8".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
match output_encoding.as_str() {
"yuv420" => {
let mut y = y.to_vec();
let mut u = u.to_vec();
let mut v = v.to_vec();
y.append(&mut u);
y.append(&mut v);
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"yuv420".to_string(),
),
);
metadata.parameters.insert(
"width".to_string(),
dora_node_api::Parameter::Integer(
p.width() as i64
),
);
metadata.parameters.insert(
"height".to_string(),
dora_node_api::Parameter::Integer(
p.height() as i64
),
);

node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
"bgr8" => {
let y = yuv420_to_bgr(
&y,
&u,
&v,
p.width(),
p.height(),
);
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"bgr8".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
_ => {
warn!(
"Unsupported output encoding {}",
output_encoding
);
continue;
}
}
}
dav1d::PixelLayout::I400 => {
let y = p.plane(dav1d::PlanarImageComponent::Y);
let vec16: Vec<u16> = bytemuck::cast_slice(&y).to_vec();
let arrow = vec16.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("mono16".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
match p.bit_depth() {
8 => {
let y = y.to_vec();
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"mono8".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
10 | 12 => {
let vec16: Vec<u16> =
bytemuck::cast_slice(&y).to_vec();
let arrow = vec16.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String(
"mono16".to_string(),
),
);
node.send_output(id, metadata.parameters, arrow)
.unwrap();
}
_ => {
warn!("Unsupported bit depth {}", p.bit_depth());
continue;
}
}
}
_ => {
warn!("Unsupported pixel layout");


+ 9
- 1
node-hub/dora-ios-lidar/dora_ios_lidar/main.py View File

@@ -12,6 +12,7 @@ from scipy.spatial.transform import Rotation

image_width = os.getenv("IMAGE_WIDTH")
image_height = os.getenv("IMAGE_HEIGHT")
ROTATE = os.getenv("ROTATE")


class DemoApp:
@@ -100,8 +101,15 @@ class DemoApp:
f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1])
r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0])
r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1])
if ROTATE == "ROTATE_90_CLOCKWISE":
rgb = cv2.rotate(rgb, cv2.ROTATE_90_CLOCKWISE)
depth = cv2.rotate(depth, cv2.ROTATE_90_CLOCKWISE)
rgb = cv2.resize(rgb, (int(image_width), int(image_height)))
depth = cv2.resize(depth, (int(image_width), int(image_height)))
depth = cv2.resize(
depth,
(int(image_width), int(image_height)),
interpolation=cv2.INTER_NEAREST,
)
else:
f_0 = intrinsic_mat[0, 0]
f_1 = intrinsic_mat[1, 1]


+ 14
- 1
node-hub/dora-rav1e/src/lib.rs View File

@@ -201,6 +201,13 @@ fn send_yuv(
metadata
.parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string()));
metadata
.parameters
.insert("height".to_string(), Parameter::Integer(enc.height as i64));
metadata
.parameters
.insert("width".to_string(), Parameter::Integer(enc.width as i64));

let data = pkt.data;
let arrow = data.into_arrow();
node.send_output(id, metadata.parameters.clone(), arrow)
@@ -274,6 +281,12 @@ pub fn lib_main() -> Result<()> {
height,
speed_settings: SpeedSettings::from_preset(speed),
low_latency: true,
chroma_sampling: color::ChromaSampling::Cs420,
color_description: Some(ColorDescription {
matrix_coefficients: MatrixCoefficients::BT709,
transfer_characteristics: color::TransferCharacteristics::BT709,
color_primaries: color::ColorPrimaries::BT709,
}),
..Default::default()
};
match encoding {
@@ -327,9 +340,9 @@ pub fn lib_main() -> Result<()> {
fill_zeros_toward_center_y_plane_in_place(&mut buffer, width, height);
}

// let buffer = shift_u16_slice_to_upper_12_bits(buffer);
let bytes: &[u8] = &bytemuck::cast_slice(&buffer);

let cfg = Config::new().with_encoder_config(enc.clone());
let mut ctx: Context<u16> = cfg.new_context().unwrap();
let mut f = ctx.new_frame();



+ 17
- 1
node-hub/opencv-plot/opencv_plot/main.py View File

@@ -1,12 +1,19 @@
"""TODO: Add docstring."""

import argparse
import io
import os

import cv2
import numpy as np
import pyarrow as pa
from dora import Node
from PIL import (
Image,
)

if True:
import pillow_avif # noqa # noqa

RUNNER_CI = True if os.getenv("CI") == "true" else False

@@ -76,6 +83,7 @@ def plot_frame(plot):

def yuv420p_to_bgr_opencv(yuv_array, width, height):
"""TODO: Add docstring."""
yuv_array = yuv_array[: width * height * 3 // 2]
yuv = yuv_array.reshape((height * 3 // 2, width))
return cv2.cvtColor(yuv, cv2.COLOR_YUV420p2RGB)

@@ -145,7 +153,6 @@ def main():
encoding = metadata["encoding"]
width = metadata["width"]
height = metadata["height"]

if encoding == "bgr8":
channels = 3
storage_type = np.uint8
@@ -181,6 +188,14 @@ def main():
img_bgr_restored = yuv420p_to_bgr_opencv(storage, width, height)

plot.frame = img_bgr_restored
elif encoding == "avif":
# Convert AVIF to RGB
array = storage.to_numpy()
bytes = array.tobytes()
img = Image.open(io.BytesIO(bytes))
img = img.convert("RGB")
plot.frame = np.array(img)
plot.frame = cv2.cvtColor(plot.frame, cv2.COLOR_RGB2BGR)
else:
raise RuntimeError(f"Unsupported image encoding: {encoding}")

@@ -188,6 +203,7 @@ def main():
if not RUNNER_CI:
if cv2.waitKey(1) & 0xFF == ord("q"):
break

elif event_id == "bbox":
arrow_bbox = event["value"][0]
bbox_format = event["metadata"]["format"]


+ 7
- 1
node-hub/opencv-plot/pyproject.toml View File

@@ -10,7 +10,13 @@ description = "Dora Node for plotting text and bbox on image with OpenCV"

requires-python = ">=3.8"

dependencies = ["dora-rs >= 0.3.9", "numpy < 2.0.0", "opencv-python >= 4.1.1"]
dependencies = [
"dora-rs >= 0.3.9",
"numpy < 2.0.0",
"opencv-python >= 4.1.1",
"pillow-avif-plugin>=1.5.1",
"pillow>=10.4.0",
]

[dependency-groups]
dev = ["pytest >=8.1.1", "ruff >=0.9.1"]


Write Preview
Loading…
Cancel
Save