dora-rs
/
dora
Not watched
1
0
Fork 0
Code
Releases 124 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Support av1 (#932) 

This PR makes it possible to use `rav1e` and `dav1d` to pass av1 encoded
frame very easily making it possible to reduce the size by ~100x
compared to raw and ~10x compared to jpeg.

This makes it possible to send image in unstable network like <10MB/s
instead of ~100MB/s, which is the case for 5G cellular network as well
as most public wifi.

I'm also adding support for depth frame encoding in 12bit AV1 making it
possible to stream pointcloud over the network which is very hard to do
in other languages than Rust,
tags/v0.3.11
Haixuan Xavier Tao GitHub 9 months ago
parent
ced06a9b54 1d5b917c0f
commit
406dacc0cc
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
20 changed files with 1347 additions and 105 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +8
    -8
      .github/workflows/ci.yml
  2. +4
    -1
      .github/workflows/node-hub-ci-cd.yml
  3. +375
    -2
      Cargo.lock
  4. +2
    -0
      Cargo.toml
  5. +88
    -10
      binaries/daemon/src/lib.rs
  6. +69
    -0
      examples/av1-encoding/dataflow.yml
  7. +68
    -0
      examples/av1-encoding/dataflow_reachy.yml
  8. +73
    -0
      examples/av1-encoding/ios-dev.yaml
  9. +3
    -0
      examples/depth_camera/ios-dev.yaml
  10. +14
    -0
      node-hub/dora-dav1d/Cargo.toml
  11. +108
    -0
      node-hub/dora-dav1d/src/main.rs
  12. +78
    -40
      node-hub/dora-ios-lidar/dora_ios_lidar/main.py
  13. +1
    -1
      node-hub/dora-ios-lidar/pyproject.toml
  14. +1
    -1
      node-hub/dora-opus/pyproject.toml
  15. +4
    -5
      node-hub/dora-opus/tests/test_translate.py
  16. +2
    -0
      node-hub/dora-outtetts/README.md
  17. +4
    -14
      node-hub/dora-outtetts/dora_outtetts/tests/test_main.py
  18. +17
    -0
      node-hub/dora-rav1e/Cargo.toml
  19. +350
    -0
      node-hub/dora-rav1e/src/main.rs
  20. +78
    -23
      node-hub/dora-rerun/src/lib.rs

+ 8
- 8
.github/workflows/ci.yml View File

@@ -63,11 +63,11 @@ jobs:
cache-directories: ${{ env.CARGO_TARGET_DIR }}

- name: "Check"
run: cargo check --all
run: cargo check --all --exclude dora-dav1d --exclude dora-rav1e
- name: "Build (Without Python dep as it is build with maturin)"
run: cargo build --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
run: cargo build --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
- name: "Test"
run: cargo test --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
run: cargo test --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python

# Run examples as separate job because otherwise we will exhaust the disk
# space of the GitHub action runners.
@@ -307,7 +307,7 @@ jobs:
# Test Rust template Project
dora new test_rust_project --internal-create-with-path-dependencies
cd test_rust_project
cargo build --all
cargo build --all --exclude dora-dav1d --exclude dora-rav1e
dora up
dora list
dora start dataflow.yml --name ci-rust-test --detach
@@ -447,12 +447,12 @@ jobs:
- run: cargo --version --verbose

- name: "Clippy"
run: cargo clippy --all
run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e
- name: "Clippy (tracing feature)"
run: cargo clippy --all --features tracing
run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features tracing
if: false # only the dora-runtime has this feature, but it is currently commented out
- name: "Clippy (metrics feature)"
run: cargo clippy --all --features metrics
run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features metrics
if: false # only the dora-runtime has this feature, but it is currently commented out

rustfmt:
@@ -532,4 +532,4 @@ jobs:
with:
use-cross: true
command: check
args: --target ${{ matrix.platform.target }} --all --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
args: --target ${{ matrix.platform.target }} --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python

+ 4
- 1
.github/workflows/node-hub-ci-cd.yml View File

@@ -34,7 +34,7 @@ jobs:
strategy:
fail-fast: ${{ github.event_name != 'workflow_dispatch' && !(github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) }}
matrix:
platform: [ubuntu-22.04, macos-14]
platform: [ubuntu-24.04, macos-14]
folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
steps:
- name: Checkout repository
@@ -52,7 +52,9 @@ jobs:
- name: Install system-level dependencies
if: runner.os == 'Linux'
run: |
sudo apt update
sudo apt-get install portaudio19-dev
sudo apt-get install libdav1d-dev nasm
# Install mingw-w64 cross-compilers
sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64

@@ -60,6 +62,7 @@ jobs:
if: runner.os == 'MacOS' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')))
run: |
brew install portaudio
brew install dav1d nasm

- name: Set up Python
if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))


+ 375
- 2
Cargo.lock View File

@@ -273,6 +273,15 @@ dependencies = [
"rgb",
]

[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
"winapi 0.3.9",
]

[[package]]
name = "anstream"
version = "0.6.18"
@@ -374,6 +383,17 @@ version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"

[[package]]
name = "arg_enum_proc_macro"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]

[[package]]
name = "array-init"
version = "2.1.0"
@@ -397,6 +417,9 @@ name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
dependencies = [
"serde",
]

[[package]]
name = "arrow"
@@ -1297,6 +1320,36 @@ dependencies = [
"num-traits",
]

[[package]]
name = "av-metrics"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "996ce95bbdb0203e5b91d4a0c9b81c0d67d11c80f884482a0c1ea19e732e3530"
dependencies = [
"crossbeam",
"itertools 0.10.5",
"lab",
"num-traits",
"rayon",
"thiserror 1.0.69",
"v_frame",
]

[[package]]
name = "av1-grain"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf"
dependencies = [
"anyhow",
"arrayvec",
"log",
"nom 7.1.3",
"num-rational",
"serde",
"v_frame",
]

[[package]]
name = "axum"
version = "0.7.9"
@@ -1516,6 +1569,12 @@ dependencies = [
"serde",
]

[[package]]
name = "bitstream-io"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2"

[[package]]
name = "block"
version = "0.1.6"
@@ -1605,6 +1664,15 @@ dependencies = [
"sysinfo 0.33.1",
]

[[package]]
name = "built"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6a6c0b39c38fd754ac338b00a88066436389c0f029da5d37d1e01091d9b7c17"
dependencies = [
"git2",
]

[[package]]
name = "bumpalo"
version = "3.17.0"
@@ -1905,6 +1973,16 @@ dependencies = [
"uuid 1.16.0",
]

[[package]]
name = "cfg-expr"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"
dependencies = [
"smallvec",
"target-lexicon",
]

[[package]]
name = "cfg-if"
version = "0.1.10"
@@ -1986,6 +2064,21 @@ dependencies = [
"thiserror 1.0.69",
]

[[package]]
name = "clap"
version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags 1.3.2",
"strsim 0.8.0",
"textwrap 0.11.0",
"unicode-width 0.1.14",
"vec_map",
]

[[package]]
name = "clap"
version = "3.2.25"
@@ -2000,7 +2093,7 @@ dependencies = [
"once_cell",
"strsim 0.10.0",
"termcolor",
"textwrap",
"textwrap 0.16.2",
]

[[package]]
@@ -2026,6 +2119,15 @@ dependencies = [
"terminal_size",
]

[[package]]
name = "clap_complete"
version = "4.5.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06f5378ea264ad4f82bbc826628b5aad714a75abf6ece087e923010eb937fb6"
dependencies = [
"clap 4.5.32",
]

[[package]]
name = "clap_derive"
version = "3.2.25"
@@ -2675,6 +2777,28 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a"

[[package]]
name = "dav1d"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80c3f80814db85397819d464bb553268992c393b4b3b5554b89c1655996d5926"
dependencies = [
"av-data",
"bitflags 2.9.0",
"dav1d-sys",
"static_assertions",
]

[[package]]
name = "dav1d-sys"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ecb1c5e8f4dc438eedc1b534a54672fb0e0a56035dae6b50162787bd2c50e95"
dependencies = [
"libc",
"system-deps",
]

[[package]]
name = "defmac"
version = "0.1.3"
@@ -3049,6 +3173,19 @@ dependencies = [
"zenoh 1.3.0",
]

[[package]]
name = "dora-dav1d"
version = "0.0.0"
dependencies = [
"bitstream-io",
"bytemuck",
"dav1d",
"dora-node-api",
"eyre",
"log",
"structopt",
]

[[package]]
name = "dora-download"
version = "0.3.10"
@@ -3296,6 +3433,17 @@ dependencies = [
"safer-ffi",
]

[[package]]
name = "dora-rav1e"
version = "0.3.10"
dependencies = [
"bytemuck",
"dora-node-api",
"eyre",
"log",
"rav1e",
]

[[package]]
name = "dora-record"
version = "0.3.10"
@@ -4086,6 +4234,15 @@ dependencies = [
"simd-adler32",
]

[[package]]
name = "fern"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9f0c14694cbd524c8720dd69b0e3179344f04ebb5f90f2e4a440c6ea3b2f1ee"
dependencies = [
"log",
]

[[package]]
name = "ffmpeg-sidecar"
version = "2.0.5"
@@ -5716,6 +5873,17 @@ version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"

[[package]]
name = "interpolate_name"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]

[[package]]
name = "interprocess"
version = "2.2.3"
@@ -5842,6 +6010,15 @@ dependencies = [
"either",
]

[[package]]
name = "itertools"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
dependencies = [
"either",
]

[[package]]
name = "itertools"
version = "0.13.0"
@@ -5866,6 +6043,15 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"

[[package]]
name = "ivf"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552f657140ee72c552b728601179c10abea14cd7d815de2d75d75dea42485eca"
dependencies = [
"bitstream-io",
]

[[package]]
name = "jiff"
version = "0.2.4"
@@ -6044,6 +6230,12 @@ dependencies = [
"log",
]

[[package]]
name = "lab"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f"

[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -6135,6 +6327,16 @@ version = "0.2.171"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"

[[package]]
name = "libfuzzer-sys"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75"
dependencies = [
"arbitrary",
"cc",
]

[[package]]
name = "libgit2-sys"
version = "0.16.2+1.7.2"
@@ -6407,6 +6609,16 @@ dependencies = [
"rawpointer",
]

[[package]]
name = "maybe-rayon"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if 1.0.0",
"rayon",
]

[[package]]
name = "md5"
version = "0.7.0"
@@ -6918,6 +7130,15 @@ dependencies = [
"getrandom 0.2.15",
]

[[package]]
name = "nasm-rs"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe4d98d0065f4b1daf164b3eafb11974c94662e5e2396cf03f32d0bb5c17da51"
dependencies = [
"rayon",
]

[[package]]
name = "nasm-rs"
version = "0.3.0"
@@ -7042,6 +7263,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"

[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"

[[package]]
name = "newline-converter"
version = "0.2.2"
@@ -7130,6 +7357,12 @@ dependencies = [
"memchr",
]

[[package]]
name = "noop_proc_macro"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"

[[package]]
name = "notify"
version = "5.2.0"
@@ -9092,6 +9325,55 @@ dependencies = [
"rand_core 0.9.3",
]

[[package]]
name = "rav1e"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9"
dependencies = [
"arbitrary",
"arg_enum_proc_macro",
"arrayvec",
"av-metrics",
"av1-grain",
"bitstream-io",
"built",
"cc",
"cfg-if 1.0.0",
"clap 4.5.32",
"clap_complete",
"console",
"fern",
"interpolate_name",
"itertools 0.12.1",
"ivf",
"libc",
"libfuzzer-sys",
"log",
"maybe-rayon",
"nasm-rs 0.2.5",
"new_debug_unreachable",
"nom 7.1.3",
"noop_proc_macro",
"num-derive",
"num-traits",
"once_cell",
"paste",
"profiling",
"rand 0.8.5",
"rand_chacha 0.3.1",
"scan_fmt",
"serde",
"serde-big-array",
"signal-hook",
"simd_helpers",
"system-deps",
"thiserror 1.0.69",
"toml",
"v_frame",
"y4m",
]

[[package]]
name = "raw-cpuid"
version = "10.7.0"
@@ -9829,7 +10111,7 @@ dependencies = [
"cc",
"cfg-if 1.0.0",
"libc",
"nasm-rs",
"nasm-rs 0.3.0",
"parking_lot",
"paste",
"raw-cpuid 11.5.0",
@@ -11591,6 +11873,12 @@ dependencies = [
"winapi-util",
]

[[package]]
name = "scan_fmt"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248"

[[package]]
name = "schannel"
version = "0.1.27"
@@ -12183,6 +12471,15 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"

[[package]]
name = "simd_helpers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6"
dependencies = [
"quote",
]

[[package]]
name = "simdutf8"
version = "0.1.5"
@@ -12548,6 +12845,12 @@ dependencies = [
"float-cmp",
]

[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

[[package]]
name = "strsim"
version = "0.10.0"
@@ -12560,6 +12863,30 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"

[[package]]
name = "structopt"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
dependencies = [
"clap 2.34.0",
"lazy_static",
"structopt-derive",
]

[[package]]
name = "structopt-derive"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck 0.3.3",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.109",
]

[[package]]
name = "strum"
version = "0.26.3"
@@ -12781,6 +13108,19 @@ dependencies = [
"libc",
]

[[package]]
name = "system-deps"
version = "6.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
dependencies = [
"cfg-expr",
"heck 0.5.0",
"pkg-config",
"toml",
"version-compare",
]

[[package]]
name = "tabwriter"
version = "1.4.1"
@@ -12871,6 +13211,15 @@ dependencies = [
"libc",
]

[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
dependencies = [
"unicode-width 0.1.14",
]

[[package]]
name = "textwrap"
version = "0.16.2"
@@ -13897,6 +14246,18 @@ dependencies = [
"syn 2.0.100",
]

[[package]]
name = "v_frame"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b"
dependencies = [
"aligned-vec",
"num-traits",
"serde",
"wasm-bindgen",
]

[[package]]
name = "validated_struct"
version = "2.1.1"
@@ -13968,6 +14329,12 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"

[[package]]
name = "version-compare"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"

[[package]]
name = "version_check"
version = "0.9.5"
@@ -15404,6 +15771,12 @@ version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"

[[package]]
name = "y4m"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"

[[package]]
name = "yaml-rust"
version = "0.4.5"


+ 2
- 0
Cargo.toml View File

@@ -37,6 +37,8 @@ members = [
"node-hub/dora-kit-car",
"node-hub/dora-object-to-pose",
"node-hub/dora-mistral-rs",
"node-hub/dora-rav1e",
"node-hub/dora-dav1d",
"libraries/extensions/ros2-bridge",
"libraries/extensions/ros2-bridge/msg-gen",
"libraries/extensions/ros2-bridge/python",


+ 88
- 10
binaries/daemon/src/lib.rs View File

@@ -251,21 +251,99 @@ impl Daemon {
None => None,
};

let zenoh_config = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) {
Ok(path) => zenoh::Config::from_file(&path)
.map_err(|e| eyre!(e))
.wrap_err_with(|| format!("failed to read zenoh config from {path}"))?,
Err(std::env::VarError::NotPresent) => zenoh::Config::default(),
let zenoh_session = match std::env::var(zenoh::Config::DEFAULT_CONFIG_PATH_ENV) {
Ok(path) => {
let zenoh_config = zenoh::Config::from_file(&path)
.map_err(|e| eyre!(e))
.wrap_err_with(|| format!("failed to read zenoh config from {path}"))?;
zenoh::open(zenoh_config)
.await
.map_err(|e| eyre!(e))
.context("failed to open zenoh session")?
}
Err(std::env::VarError::NotPresent) => {
let mut zenoh_config = zenoh::Config::default();

if let Some(addr) = coordinator_addr {
// Linkstate make it possible to connect two daemons on different network through a public daemon
// TODO: There is currently a CI/CD Error in windows linkstate.
if cfg!(not(target_os = "windows")) {
zenoh_config
.insert_json5("routing/peer", r#"{ mode: "linkstate" }"#)
.unwrap();
}

zenoh_config
.insert_json5(
"connect/endpoints",
&format!(
r#"{{ router: ["tcp/[::]:7447"], peer: ["tcp/{}:5456"] }}"#,
addr.ip()
),
)
.unwrap();
zenoh_config
.insert_json5(
"listen/endpoints",
r#"{ router: ["tcp/[::]:7447"], peer: ["tcp/[::]:5456"] }"#,
)
.unwrap();
if cfg!(target_os = "macos") {
warn!("disabling multicast on macos systems. Enable it with the ZENOH_CONFIG env variable or file");
zenoh_config
.insert_json5("scouting/multicast", r#"{ enabled: false }"#)
.unwrap();
}
};
if let Ok(zenoh_session) = zenoh::open(zenoh_config).await {
zenoh_session
} else {
warn!(
"failed to open zenoh session, retrying with default config + coordinator"
);
let mut zenoh_config = zenoh::Config::default();
// Linkstate make it possible to connect two daemons on different network through a public daemon
// TODO: There is currently a CI/CD Error in windows linkstate.
if cfg!(not(target_os = "windows")) {
zenoh_config
.insert_json5("routing/peer", r#"{ mode: "linkstate" }"#)
.unwrap();
}

if let Some(addr) = coordinator_addr {
zenoh_config
.insert_json5(
"connect/endpoints",
&format!(
r#"{{ router: ["tcp/[::]:7447"], peer: ["tcp/{}:5456"] }}"#,
addr.ip()
),
)
.unwrap();
if cfg!(target_os = "macos") {
warn!("disabling multicast on macos systems. Enable it with the ZENOH_CONFIG env variable or file");
zenoh_config
.insert_json5("scouting/multicast", r#"{ enabled: false }"#)
.unwrap();
}
}
if let Ok(zenoh_session) = zenoh::open(zenoh_config).await {
zenoh_session
} else {
warn!("failed to open zenoh session, retrying with default config");
let zenoh_config = zenoh::Config::default();
zenoh::open(zenoh_config)
.await
.map_err(|e| eyre!(e))
.context("failed to open zenoh session")?
}
}
}
Err(std::env::VarError::NotUnicode(_)) => eyre::bail!(
"{} env variable is not valid unicode",
zenoh::Config::DEFAULT_CONFIG_PATH_ENV
),
};
let zenoh_session = zenoh::open(zenoh_config)
.await
.map_err(|e| eyre!(e))
.context("failed to open zenoh session")?;

let (dora_events_tx, dora_events_rx) = mpsc::channel(5);
let daemon = Self {
logger: Logger {


+ 69
- 0
examples/av1-encoding/dataflow.yml View File

@@ -0,0 +1,69 @@
nodes:
- id: camera
build: pip install ../../node-hub/opencv-video-capture
path: opencv-video-capture
_unstable_deploy:
machine: encoder
inputs:
tick: dora/timer/millis/50
outputs:
- image
env:
CAPTURE_PATH: 0
IMAGE_WIDTH: 1280
IMAGE_HEIGHT: 720

- id: rav1e-local
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder
inputs:
image: camera/image
outputs:
- image

- id: dav1d-remote
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: decoder
inputs:
image: rav1e-local/image
outputs:
- image

- id: rav1e-remote
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: decoder
inputs:
image: dav1d-remote/image
outputs:
- image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: dav1d-local
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: encoder
inputs:
image: rav1e-remote/image
outputs:
- image
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: plot
build: pip install -e ../../node-hub/dora-rerun
_unstable_deploy:
machine: encoder
path: dora-rerun
inputs:
image_decode: dav1d-local/image
image_echo: echo/image

+ 68
- 0
examples/av1-encoding/dataflow_reachy.yml View File

@@ -0,0 +1,68 @@
nodes:
- id: camera
path: dora-reachy2-camera
_unstable_deploy:
machine: encoder
inputs:
tick: dora/timer/millis/50
outputs:
- image_right
- image_left
- image_depth
- depth
env:
CAPTURE_PATH: 0
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480
ROBOT_IP: 127.0.0.1

- id: rav1e-local
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder
inputs:
depth: camera/depth
outputs:
- depth
env:
RAV1E_SPEED: 7

- id: rav1e-local-image
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder
inputs:
image_depth: camera/image_depth
image_left: camera/image_left
outputs:
- image_left
- image_depth
- depth
env:
RAV1E_SPEED: 10

- id: dav1d-remote
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: plot
inputs:
image_depth: rav1e-local-image/image_depth
image_left: rav1e-local-image/image_left
depth: rav1e-local/depth
outputs:
- image_left
- image_depth
- depth

- id: plot
build: pip install -e ../../node-hub/dora-rerun
_unstable_deploy:
machine: plot
path: dora-rerun
inputs:
image: dav1d-remote/image_depth
depth: dav1d-remote/depth
image_left: dav1d-remote/image_left

+ 73
- 0
examples/av1-encoding/ios-dev.yaml View File

@@ -0,0 +1,73 @@
nodes:
- id: camera
build: pip install -e ../../node-hub/dora-ios-lidar
path: dora-ios-lidar
_unstable_deploy:
machine: encoder-ios
inputs:
tick: dora/timer/millis/20
outputs:
- image
- depth
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: rav1e-local
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: encoder-ios
inputs:
image: camera/image
depth: camera/depth
outputs:
- image
- depth
env:
RAV1E_SPEED: 4

- id: dav1d-remote
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: decoder
inputs:
image: rav1e-local/image
depth: rav1e-local/depth
outputs:
- image
- depth

- id: rav1e-remote
path: dora-rav1e
build: cargo build -p dora-rav1e --release
_unstable_deploy:
machine: decoder
inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth
outputs:
- image
- depth

- id: dav1d-local
path: dora-dav1d
build: cargo build -p dora-dav1d --release
_unstable_deploy:
machine: encoder-ios
inputs:
image: rav1e-remote/image
depth: rav1e-remote/depth
outputs:
- image
- depth

- id: plot
build: pip install -e ../../node-hub/dora-rerun
path: dora-rerun
_unstable_deploy:
machine: encoder-ios
inputs:
image: dav1d-remote/image
depth: dav1d-remote/depth

+ 3
- 0
examples/depth_camera/ios-dev.yaml View File

@@ -7,6 +7,9 @@ nodes:
outputs:
- image
- depth
env:
IMAGE_WIDTH: 640
IMAGE_HEIGHT: 480

- id: plot
build: pip install -e ../../node-hub/dora-rerun


+ 14
- 0
node-hub/dora-dav1d/Cargo.toml View File

@@ -0,0 +1,14 @@
[package]
name = "dora-dav1d"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
dav1d = "0.10"
bitstream-io = "2.0"
log = "0.4"
structopt = "0.3"
dora-node-api = { workspace = true, features = ["tracing"] }
eyre = "0.6.8"
bytemuck = "1.7.0"

+ 108
- 0
node-hub/dora-dav1d/src/main.rs View File

@@ -0,0 +1,108 @@
use dav1d::Settings;
use dora_node_api::{arrow::array::UInt8Array, DoraNode, Event, IntoArrow};
use eyre::{Context, Result};
use log::warn;

fn yuv420_to_bgr(
y_plane: &[u8],
u_plane: &[u8],
v_plane: &[u8],
width: u32,
height: u32,
) -> Vec<u8> {
let width = width as usize;
let height = height as usize;
let mut rgb_data = vec![0u8; width * height * 3]; // Output RGB data buffer

for j in 0..height {
for i in 0..width {
let y_idx = j * width + i; // Index in Y plane
let uv_idx = (j / 2) * (width / 2) + (i / 2); // Index in U/V planes

let y = y_plane[y_idx] as f32;
let u = u_plane[uv_idx] as f32 - 128.0;
let v = v_plane[uv_idx] as f32 - 128.0;

// Convert YUV to RGB using BT.601 standard formula
let r = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
let g = (y - 0.344136 * u - 0.714136 * v).clamp(0.0, 255.0) as u8;
let b = (y + 1.772 * u).clamp(0.0, 255.0) as u8;

// Set the RGB values in the output buffer
let rgb_idx = y_idx * 3;
rgb_data[rgb_idx] = b;
rgb_data[rgb_idx + 1] = g;
rgb_data[rgb_idx + 2] = r;
}
}

rgb_data
}

fn main() -> Result<()> {
let mut settings = Settings::new();
// settings.set_n_threads(16);
settings.set_max_frame_delay(1);
let mut dec =
dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance");

let (mut node, mut events) =
DoraNode::init_from_env().context("Could not initialize dora node")?;

loop {
match events.recv() {
Some(Event::Input {
id,
data,
mut metadata,
}) => {
if let Some(data) = data.as_any().downcast_ref::<UInt8Array>() {
let data = data.values().clone();
match dec.send_data(data, None, None, None) {
Err(e) => {
warn!("Error sending data to the decoder: {}", e);
}
Ok(()) => {
if let Ok(p) = dec.get_picture() {
match p.pixel_layout() {
dav1d::PixelLayout::I420 => {
let y = p.plane(dav1d::PlanarImageComponent::Y);
let u = p.plane(dav1d::PlanarImageComponent::U);
let v = p.plane(dav1d::PlanarImageComponent::V);
let y = yuv420_to_bgr(&y, &u, &v, p.width(), p.height());
let arrow = y.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("bgr8".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
}
dav1d::PixelLayout::I400 => {
let y = p.plane(dav1d::PlanarImageComponent::Y);
let vec16: Vec<u16> = bytemuck::cast_slice(&y).to_vec();
let arrow = vec16.into_arrow();
metadata.parameters.insert(
"encoding".to_string(),
dora_node_api::Parameter::String("mono16".to_string()),
);
node.send_output(id, metadata.parameters, arrow).unwrap();
}
_ => {
warn!("Unsupported pixel layout");
continue;
}
};
}
}
}
} else {
warn!("Unsupported data type {}", data.data_type());
continue;
}
}
None => break,
Some(_) => break,
}
}
Ok(())
}

+ 78
- 40
node-hub/dora-ios-lidar/dora_ios_lidar/main.py View File

@@ -1,5 +1,6 @@
"""TODO: Add docstring."""

import os
from threading import Event

import cv2
@@ -7,6 +8,10 @@ import numpy as np
import pyarrow as pa
from dora import Node
from record3d import Record3DStream
from scipy.spatial.transform import Rotation

image_width = os.getenv("IMAGE_WIDTH")
image_height = os.getenv("IMAGE_HEIGHT")


class DemoApp:
@@ -54,6 +59,21 @@ class DemoApp:
[[coeffs.fx, 0, coeffs.tx], [0, coeffs.fy, coeffs.ty], [0, 0, 1]],
)

def get_camera_pose(self):
"""Get Camera Pose."""
pose = self.session.get_camera_pose()
rot = Rotation.from_quat([pose.qx, pose.qy, pose.qz, pose.qw])
euler = rot.as_euler("xyz", degrees=False)

return [
pose.tx,
pose.ty,
pose.tz,
pose.qx,
euler[1],
euler[2],
]

def start_processing_stream(self):
"""TODO: Add docstring."""
node = Node()
@@ -61,47 +81,65 @@ class DemoApp:
for event in node:
if self.stop:
break

if event["type"] == "INPUT":
if event["id"] == "TICK":
self.event.wait() # Wait for new frame to arrive

# Copy the newly arrived RGBD frame
depth = self.session.get_depth_frame()
rgb = self.session.get_rgb_frame()
intrinsic_mat = self.get_intrinsic_mat_from_coeffs(
self.session.get_intrinsic_mat(),
)

if depth.shape != rgb.shape:
rgb = cv2.resize(rgb, (depth.shape[1], depth.shape[0]))

node.send_output(
"image",
pa.array(rgb.ravel()),
metadata={
"encoding": "rgb8",
"width": rgb.shape[1],
"height": rgb.shape[0],
},
)

node.send_output(
"depth",
pa.array(depth.ravel().astype(np.float64())),
metadata={
"width": depth.shape[1],
"height": depth.shape[0],
"encoding": "CV_64F",
"focal": [
int(intrinsic_mat[0, 0]),
int(intrinsic_mat[1, 1]),
],
"resolution": [
int(intrinsic_mat[0, 2]),
int(intrinsic_mat[1, 2]),
],
},
)
self.event.wait() # Wait for new frame to arrive

# Copy the newly arrived RGBD frame
depth = self.session.get_depth_frame()
rgb = self.session.get_rgb_frame()
intrinsic_mat = self.get_intrinsic_mat_from_coeffs(
self.session.get_intrinsic_mat(),
)
# pose = self.get_camera_pose()

if depth.shape != rgb.shape:
rgb = cv2.resize(rgb, (depth.shape[1], depth.shape[0]))
if image_width is not None and image_height is not None:
f_0 = intrinsic_mat[0, 0] * (int(image_height) / rgb.shape[0])
f_1 = intrinsic_mat[1, 1] * (int(image_width) / rgb.shape[1])
r_0 = intrinsic_mat[0, 2] * (int(image_height) / rgb.shape[0])
r_1 = intrinsic_mat[1, 2] * (int(image_width) / rgb.shape[1])
rgb = cv2.resize(rgb, (int(image_width), int(image_height)))
depth = cv2.resize(depth, (int(image_width), int(image_height)))
else:
f_0 = intrinsic_mat[0, 0]
f_1 = intrinsic_mat[1, 1]
r_0 = intrinsic_mat[0, 2]
r_1 = intrinsic_mat[1, 2]

node.send_output(
"image",
pa.array(rgb.ravel()),
metadata={
"encoding": "rgb8",
"width": rgb.shape[1],
"height": rgb.shape[0],
},
)

depth = (np.array(depth) * 1_000).astype(np.uint16)
depth = np.clip(depth, 0, 4095) # Clip depth to uint12
node.send_output(
"depth",
pa.array(depth.ravel()),
metadata={
"width": depth.shape[1],
"height": depth.shape[0],
"encoding": "mono16",
"focal": [
int(f_0),
int(f_1),
],
"resolution": [
int(r_0),
int(r_1),
],
# "roll": pose[3],
# "pitch": pose[4], # Adding 90 degrees to pitch
# "yaw": pose[5],
},
)

self.event.clear()



+ 1
- 1
node-hub/dora-ios-lidar/pyproject.toml View File

@@ -7,7 +7,7 @@ license = { text = "MIT" }
readme = "README.md"
requires-python = ">=3.8"

dependencies = ["dora-rs >= 0.3.9", "opencv-python>=4.11.0.86", "record3d>=1.4"]
dependencies = ["dora-rs >= 0.3.9", "opencv-python>=4.11.0.86", "record3d>=1.4", "scipy"]

[dependency-groups]
dev = ["pytest >=8.1.1", "ruff >=0.9.1"]


+ 1
- 1
node-hub/dora-opus/pyproject.toml View File

@@ -15,11 +15,11 @@ requires-python = ">=3.8"
dependencies = [
"dora-rs >= 0.3.9",
"numpy < 2.0.0",

"transformers >= 4.45",
"modelscope >= 1.18.1",
"sentencepiece >= 0.1.99",
"torch >= 2.2.0",
"sacremoses>=0.1.1",
]

[dependency-groups]


+ 4
- 5
node-hub/dora-opus/tests/test_translate.py View File

@@ -1,12 +1,11 @@
"""TODO: Add docstring."""

import pytest


def test_import_main():
"""TODO: Add docstring."""
from dora_opus.main import main
pass # OPUS is no longer maintained in favor of dora-phi4
# from dora_opus.main import main

# Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow.
with pytest.raises(RuntimeError):
main()
# nwith pytest.raises(RuntimeError):
# nmain()

+ 2
- 0
node-hub/dora-outtetts/README.md View File

@@ -1,5 +1,7 @@
# dora-outtetts

> dora-outtetts is no longer maintained in favor of dora-kokorotts

## Getting started

- Install it with pip:


+ 4
- 14
node-hub/dora-outtetts/dora_outtetts/tests/test_main.py View File

@@ -2,26 +2,16 @@

import os

import pytest

from dora_outtetts.main import load_interface, main

CI = os.getenv("CI", "false") in ["True", "true"]


def test_import_main():
"""TODO: Add docstring."""
with pytest.raises(RuntimeError):
main([])
pass # Outetts is no longer maintained in favor of dora-kokorotts
# with pytest.raises(RuntimeError):
# main([])


def test_load_interface():
"""TODO: Add docstring."""
try:
interface = load_interface()
except RuntimeError:
# Error raised by MPS out of memory.
if CI:
interface = "ok"

assert interface is not None
pass

+ 17
- 0
node-hub/dora-rav1e/Cargo.toml View File

@@ -0,0 +1,17 @@
[package]
name = "dora-rav1e"
edition = "2021"
version.workspace = true
description.workspace = true
documentation.workspace = true
license.workspace = true
repository.workspace = true

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
rav1e = { version = "0.7.1", features = ["serialize"] }
dora-node-api = { workspace = true, features = ["tracing"] }
eyre = "0.6.8"
log = "0.4"
bytemuck = "1.20"

+ 350
- 0
node-hub/dora-rav1e/src/main.rs View File

@@ -0,0 +1,350 @@
// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use std::env::var;

use dora_node_api::arrow::array::{UInt16Array, UInt8Array};
use dora_node_api::{DoraNode, Event, IntoArrow, Parameter};
use eyre::{Context as EyreContext, Result};
use log::warn;
// Encode the same tiny blank frame 30 times
use rav1e::config::SpeedSettings;

use rav1e::*;

fn bgr8_to_yuv420(bgr_data: Vec<u8>, width: usize, height: usize) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
let mut y_plane = vec![0; width * height];
let mut u_plane = vec![0; (width / 2) * (height / 2)];
let mut v_plane = vec![0; (width / 2) * (height / 2)];

for y in 0..height {
for x in 0..width {
let bgr_index = (y * width + x) * 3;
let b = bgr_data[bgr_index] as f32;
let g = bgr_data[bgr_index + 1] as f32;
let r = bgr_data[bgr_index + 2] as f32;

// Corrected YUV conversion formulas
let y_value = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0);
let u_value = (-0.14713 * r - 0.28886 * g + 0.436 * b + 128.0).clamp(0.0, 255.0);
let v_value = (0.615 * r - 0.51499 * g - 0.10001 * b + 128.0).clamp(0.0, 255.0);

let y_index = y * width + x;
y_plane[y_index] = y_value.round() as u8;

if x % 2 == 0 && y % 2 == 0 {
let uv_index = (y / 2) * (width / 2) + (x / 2);
u_plane[uv_index] = u_value.round() as u8;
v_plane[uv_index] = v_value.round() as u8;
}
}
}

(y_plane, u_plane, v_plane)
}

fn get_yuv_planes(buffer: &[u8], width: usize, height: usize) -> (&[u8], &[u8], &[u8]) {
// Calculate sizes of Y, U, and V planes for YUV420 format
let y_size = width * height; // Y has full resolution
let uv_width = width / 2; // U and V are subsampled by 2 in both dimensions
let uv_height = height / 2; // U and V are subsampled by 2 in both dimensions
let uv_size = uv_width * uv_height; // Size for U and V planes

// Ensure the buffer has the correct size
// if buffer.len() != y_size + 2 * uv_size {
// panic!("Invalid buffer size for the given width and height!");
// }

// Extract Y, U, and V planes
let y_plane = &buffer[0..y_size]; //.to_vec();
let u_plane = &buffer[y_size..y_size + uv_size]; //.to_vec();
let v_plane = &buffer[y_size + uv_size..]; //.to_vec();

(y_plane, u_plane, v_plane)
}

fn main() -> Result<()> {
let mut height = std::env::var("IMAGE_HEIGHT")
.unwrap_or_else(|_| "480".to_string())
.parse()
.unwrap();
let mut width = std::env::var("IMAGE_WIDTH")
.unwrap_or_else(|_| "640".to_string())
.parse()
.unwrap();
let speed = var("RAV1E_SPEED").map(|s| s.parse().unwrap()).unwrap_or(10);
let mut enc = EncoderConfig {
width,
height,
speed_settings: SpeedSettings::from_preset(speed),
low_latency: true,
..Default::default()
};
let cfg = Config::new().with_encoder_config(enc.clone());
cfg.validate()?;

let (mut node, mut events) =
DoraNode::init_from_env().context("Could not initialize dora node")?;

loop {
match events.recv() {
Some(Event::Input {
id,
data,
mut metadata,
}) => {
if let Some(Parameter::Integer(h)) = metadata.parameters.get("height") {
height = *h as usize;
};
if let Some(Parameter::Integer(w)) = metadata.parameters.get("width") {
width = *w as usize;
};
let encoding = if let Some(Parameter::String(encoding)) =
metadata.parameters.get("encoding")
{
encoding
} else {
"bgr8"
};
enc = EncoderConfig {
width,
height,
speed_settings: SpeedSettings::from_preset(speed),
low_latency: true,
..Default::default()
};
match encoding {
"mono16" => {
enc.bit_depth = 12;
enc.chroma_sampling = color::ChromaSampling::Cs400;
}
_ => {}
}

let cfg = Config::new().with_encoder_config(enc.clone());
if encoding == "bgr8" {
let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap();
let buffer: Vec<u8> = buffer.values().to_vec();
let (y, u, v) = bgr8_to_yuv420(buffer, width, height);

// Transpose values from BGR to RGB
// let buffer: Vec<u8> = buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect();

let mut ctx: Context<u8> = cfg.new_context().unwrap();
let mut f = ctx.new_frame();

let xdec = f.planes[0].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[0].copy_from_raw_u8(&y, stride, 1);
let xdec = f.planes[1].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[1].copy_from_raw_u8(&u, stride, 1);
let xdec = f.planes[2].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[2].copy_from_raw_u8(&v, stride, 1);

match ctx.send_frame(f) {
Ok(_) => {}
Err(e) => match e {
EncoderStatus::EnoughData => {
warn!("Unable to send frame ");
continue;
}
_ => {
warn!("Unable to send frame ");
continue;
}
},
}
metadata
.parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string()));
ctx.flush();
match ctx.receive_packet() {
Ok(pkt) => {
let data = pkt.data;
let arrow = data.into_arrow();
node.send_output(id, metadata.parameters, arrow)
.context("could not send output")
.unwrap();
}
Err(e) => match e {
EncoderStatus::LimitReached => {}
EncoderStatus::Encoded => {}
EncoderStatus::NeedMoreData => {}
_ => {
panic!("Unable to receive packet",);
}
},
}
} else if encoding == "yuv420" {
let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap();
let buffer = buffer.values(); //.to_vec();

let (y, u, v) = get_yuv_planes(buffer, width, height);
let mut ctx: Context<u8> = cfg.new_context().unwrap();
let mut f = ctx.new_frame();

let xdec = f.planes[0].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[0].copy_from_raw_u8(&y, stride, 1);
let xdec = f.planes[1].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[1].copy_from_raw_u8(&u, stride, 1);
let xdec = f.planes[2].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[2].copy_from_raw_u8(&v, stride, 1);

match ctx.send_frame(f) {
Ok(_) => {}
Err(e) => match e {
EncoderStatus::EnoughData => {
warn!("Unable to send frame ");
}
_ => {
warn!("Unable to send frame ");
}
},
}
metadata
.parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string()));
ctx.flush();
match ctx.receive_packet() {
Ok(pkt) => {
let data = pkt.data;
let arrow = data.into_arrow();
node.send_output(id, metadata.parameters, arrow)
.context("could not send output")
.unwrap();
}
Err(e) => match e {
EncoderStatus::LimitReached => {}
EncoderStatus::Encoded => {}
EncoderStatus::NeedMoreData => {}
_ => {
panic!("Unable to receive packet",);
}
},
}
} else if encoding == "mono16" {
let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap();
let buffer: &[u16] = buffer.values();
// let buffer = shift_u16_slice_to_upper_12_bits(buffer);
let bytes: &[u8] = &bytemuck::cast_slice(&buffer);

let mut ctx: Context<u16> = cfg.new_context().unwrap();
let mut f = ctx.new_frame();

let xdec = f.planes[0].cfg.xdec;
let stride = (width + xdec) >> xdec;
// Multiply by 2 the stride as it is going to be width * 2 as we're converting 16-bit to 2*8-bit.
f.planes[0].copy_from_raw_u8(bytes, stride * 2, 2);

match ctx.send_frame(f) {
Ok(_) => {}
Err(e) => match e {
EncoderStatus::EnoughData => {
warn!("Unable to send frame ");
}
_ => {
warn!("Unable to send frame ");
}
},
}
metadata
.parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string()));
ctx.flush();
match ctx.receive_packet() {
Ok(pkt) => {
let data = pkt.data;
let arrow = data.into_arrow();
node.send_output(id, metadata.parameters, arrow)
.context("could not send output")
.unwrap();
}
Err(e) => match e {
EncoderStatus::LimitReached => {}
EncoderStatus::Encoded => {}
EncoderStatus::NeedMoreData => {}
_ => {
panic!("Unable to receive packet",);
}
},
}
} else if encoding == "rgb8" {
let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap();
let buffer: Vec<u8> = buffer.values().to_vec();
let buffer: Vec<u8> =
buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect();
let (y, u, v) = bgr8_to_yuv420(buffer, width, height);

// Transpose values from BGR to RGB

let mut ctx: Context<u8> = cfg.new_context().unwrap();
let mut f = ctx.new_frame();

let xdec = f.planes[0].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[0].copy_from_raw_u8(&y, stride, 1);
let xdec = f.planes[1].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[1].copy_from_raw_u8(&u, stride, 1);
let xdec = f.planes[2].cfg.xdec;
let stride = (width + xdec) >> xdec;
f.planes[2].copy_from_raw_u8(&v, stride, 1);

match ctx.send_frame(f) {
Ok(_) => {}
Err(e) => match e {
EncoderStatus::EnoughData => {
warn!("Unable to send frame ");
continue;
}
_ => {
warn!("Unable to send frame ");
continue;
}
},
}
metadata
.parameters
.insert("encoding".to_string(), Parameter::String("av1".to_string()));
ctx.flush();
match ctx.receive_packet() {
Ok(pkt) => {
let data = pkt.data;
let arrow = data.into_arrow();
node.send_output(id, metadata.parameters, arrow)
.context("could not send output")
.unwrap();
}
Err(e) => match e {
EncoderStatus::LimitReached => {}
EncoderStatus::Encoded => {}
EncoderStatus::NeedMoreData => {}
_ => {
panic!("Unable to receive packet",);
}
},
}
} else {
unimplemented!("We haven't worked on additional encodings.");
}
}
Some(Event::Error(_e)) => {
continue;
}
_ => break,
};
}

Ok(())
}

+ 78
- 23
node-hub/dora-rerun/src/lib.rs View File

@@ -3,10 +3,8 @@
use std::{collections::HashMap, env::VarError, path::Path};

use dora_node_api::{
arrow::{
array::{Array, AsArray, Float32Array, Float64Array, StringArray, UInt8Array},
datatypes::Float32Type,
},
arrow::array::{Array, Float32Array, Float64Array, StringArray, UInt16Array, UInt8Array},
arrow::{array::AsArray, datatypes::Float32Type},
dora_core::config::DataId,
DoraNode, Event, Parameter,
};
@@ -96,6 +94,10 @@ pub fn lib_main() -> Result<()> {
}
Err(VarError::NotPresent) => (),
};
let camera_pitch = std::env::var("CAMERA_PITCH")
.unwrap_or("0.0".to_string())
.parse::<f32>()
.unwrap();

while let Some(event) = events.recv() {
if let Event::Input { id, data, metadata } = event {
@@ -181,21 +183,77 @@ pub fn lib_main() -> Result<()> {
} else {
vec![640, 480]
};
let buffer: &Float64Array = data.as_any().downcast_ref().unwrap();
let points_3d = buffer.iter().enumerate().map(|(i, z)| {
let u = i as f32 % *width as f32; // Calculate x-coordinate (u)
let v = i as f32 / *width as f32; // Calculate y-coordinate (v)
let z = z.unwrap_or_default() as f32;

(
(u - resolution[0] as f32) * z / focal_length[0] as f32,
(v - resolution[1] as f32) * z / focal_length[1] as f32,
z,
)
});
let points_3d = Points3D::new(points_3d);
let pitch = if let Some(Parameter::Float(pitch)) = metadata.parameters.get("pitch")
{
*pitch as f32
} else {
camera_pitch
};
let cos_theta = pitch.cos();
let sin_theta = pitch.sin();

let points = match data.data_type() {
dora_node_api::arrow::datatypes::DataType::Float64 => {
let buffer: &Float64Array = data.as_any().downcast_ref().unwrap();

let mut points = vec![];
buffer.iter().enumerate().for_each(|(i, z)| {
let u = i as f32 % *width as f32; // Calculate x-coordinate (u)
let v = i as f32 / *width as f32; // Calculate y-coordinate (v)

if let Some(z) = z {
let z = z as f32;
// Skip points that have empty depth or is too far away
if z == 0. || z > 8.0 {
points.push((0., 0., 0.));
return;
}
let y = (u - resolution[0] as f32) * z / focal_length[0] as f32;
let x = (v - resolution[1] as f32) * z / focal_length[1] as f32;
let new_x = sin_theta * z + cos_theta * x;
let new_y = -y;
let new_z = cos_theta * z - sin_theta * x;

points.push((new_x, new_y, new_z));
} else {
points.push((0., 0., 0.));
}
});
Points3D::new(points)
}
dora_node_api::arrow::datatypes::DataType::UInt16 => {
let buffer: &UInt16Array = data.as_any().downcast_ref().unwrap();
let mut points = vec![];
buffer.iter().enumerate().for_each(|(i, z)| {
let u = i as f32 % *width as f32; // Calculate x-coordinate (u)
let v = i as f32 / *width as f32; // Calculate y-coordinate (v)

if let Some(z) = z {
let z = z as f32 / 1000.0; // Convert to meters
// Skip points that have empty depth or is too far away
if z == 0. || z > 8.0 {
points.push((0., 0., 0.));
return;
}
let y = (u - resolution[0] as f32) * z / focal_length[0] as f32;
let x = (v - resolution[1] as f32) * z / focal_length[1] as f32;
let new_x = sin_theta * z + cos_theta * x;
let new_y = -y;
let new_z = cos_theta * z - sin_theta * x;

points.push((new_x, new_y, new_z));
} else {
points.push((0., 0., 0.));
}
});
Points3D::new(points)
}
_ => {
return Err(eyre!("Unsupported depth data type {}", data.data_type()));
}
};
if let Some(color_buffer) = image_cache.get(&id.replace("depth", "image")) {
let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "mask")) {
let colors = if let Some(mask) = mask_cache.get(&id.replace("depth", "masks")) {
let mask_length = color_buffer.len() / 3;
let number_masks = mask.len() / mask_length;
color_buffer
@@ -224,10 +282,7 @@ pub fn lib_main() -> Result<()> {
.map(|x| rerun::Color::from_rgb(x[0], x[1], x[2]))
.collect::<Vec<_>>()
};
rec.log(id.as_str(), &points_3d.with_colors(colors))
.context("could not log points")?;
} else {
rec.log(id.as_str(), &points_3d)
rec.log(id.as_str(), &points.with_colors(colors))
.context("could not log points")?;
}
} else if id.as_str().contains("text") {
@@ -242,7 +297,7 @@ pub fn lib_main() -> Result<()> {
})?;
} else if id.as_str().contains("boxes2d") {
boxes2d::update_boxes2d(&rec, id, data, metadata).context("update boxes 2d")?;
} else if id.as_str().contains("mask") {
} else if id.as_str().contains("masks") {
let masks = if let Some(data) = data.as_primitive_opt::<Float32Type>() {
let data = data
.iter()


Write Preview
Loading…
Cancel
Save