.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,31 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Dora start daemon: `dora up` 
-2. Start a new dataflow: `dora start dataflow.yaml`
-3. Stop dataflow: `dora stop`
-4. Destroy dataflow: `dora destroy`
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots or Video**
-If applicable, add screenshots to help explain your problem.
-
-**Environments (please complete the following information):**
- - System info: [use `uname --all` on Linux]
- - Dora version: [use `dora --version` and `pip show dora-rs`]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/labeler.yml

@@ -1,42 +0,0 @@
-# Add/remove 'critical' label if issue contains the words 'urgent' or 'critical'
-critical:
-  - "(critical|urgent)"
-
-cli:
-  - "/(cli|command)/i"
-
-daemon:
-  - "daemon"
-
-coordinator:
-  - "coordinator"
-
-runtime:
-  - "runtime"
-
-python:
-  - "python"
-
-c:
-  - "/\bc\b/i"
-
-c++:
-  - "cxx"
-
-rust:
-  - "rust"
-
-windows:
-  - "windows"
-
-macos:
-  - "macos"
-
-linux:
-  - "(linux|ubuntu)"
-
-bug:
-  - "bug"
-
-documentation:
-  - "(doc|documentation)"

.github/renovate.json

@@ -1,21 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended",
-    ":semanticCommitTypeAll(chore)"
-  ],
-  "rangeStrategy": "in-range-only",
-  "packageRules": [
-    {
-      "matchUpdateTypes": [
-        "lockFileMaintenance",
-        "patch",
-        "minor"
-      ],
-      "groupName": "dependencies"
-    }
-  ],
-  "schedule": [
-    "on monday"
-  ]
-}

.github/workflows/cargo-release.yml

@@ -1,104 +0,0 @@
-name: Cargo Release
-
-permissions:
-  contents: write
-
-on:
-  release:
-    types:
-      - "published"
-  workflow_dispatch:
-
-jobs:
-  cargo-release:
-    name: "Cargo Release"
-
-    strategy:
-      matrix:
-        platform: [ubuntu-22.04]
-      fail-fast: false
-    runs-on: ${{ matrix.platform }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        if: runner.os == 'Linux'
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: true
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
-
-      - name: "Publish packages on `crates.io`"
-        if: runner.os == 'Linux'
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: |
-
-          # Publishing those crates from outer crates with no dependency to inner crates
-          # As cargo is going to rebuild the crates based on published dependencies
-          # we need to publish those outer crates first to be able to test the publication
-          # of inner crates.
-          #
-          # We should preferably test pre-releases before testing releases as
-          # cargo publish might catch release issues that the workspace manages to fix using
-          # workspace crates.
-          publish_if_not_exists() {
-              local package_name=$1
-              local version=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="'"$package_name"'") | .version')
-
-              if [[ -z $version ]]; then
-                  echo "error: package '$package_name' not found in the workspace."
-                  return 1
-              fi
-
-              if cargo search "$package_name" | grep -q "^$package_name = \"$version\""; then
-                  echo "package '$package_name' version '$version' already exists on crates.io. skipping publish."
-              else
-                  echo "publishing package '$package_name' version '$version'..."
-                  cargo publish --package "$package_name"
-              fi
-          }
-
-          # the dora-message package is versioned separately, so this publish command might fail if the version is already published
-          publish_if_not_exists dora-message
-
-          # Publish libraries crates
-          publish_if_not_exists dora-tracing
-          publish_if_not_exists dora-metrics
-          publish_if_not_exists dora-download
-          publish_if_not_exists dora-core
-          publish_if_not_exists communication-layer-pub-sub
-          publish_if_not_exists communication-layer-request-reply
-          publish_if_not_exists shared-memory-server
-          publish_if_not_exists dora-arrow-convert
-
-          # Publish rust API
-          publish_if_not_exists dora-operator-api-macros
-          publish_if_not_exists dora-operator-api-types
-          publish_if_not_exists dora-operator-api
-          publish_if_not_exists dora-node-api
-          publish_if_not_exists dora-operator-api-python
-          publish_if_not_exists dora-operator-api-c
-          publish_if_not_exists dora-node-api-c
-
-          # Publish binaries crates
-          publish_if_not_exists dora-coordinator
-          publish_if_not_exists dora-runtime
-          publish_if_not_exists dora-daemon
-          publish_if_not_exists dora-cli
-
-          # Publish ROS2 bridge
-          publish_if_not_exists dora-ros2-bridge-msg-gen
-          publish_if_not_exists dora-ros2-bridge

.github/workflows/ci.yml

@@ -1,557 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - "node-hub/**"
-  workflow_dispatch:
-
-env:
-  RUST_LOG: INFO
-
-jobs:
-  test:
-    name: "Test"
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-      fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    timeout-minutes: 60
-    steps:
-      - name: Print available space (Windows only)
-        run: Get-PSDrive
-        if: runner.os == 'Windows'
-      - name: Override cargo target dir (Windows only)
-        run: echo "CARGO_TARGET_DIR=C:\cargo-target" >> "$GITHUB_ENV"
-        shell: bash
-        if: runner.os == 'Windows'
-
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        if: runner.os == 'Linux'
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: false
-      - name: Free disk Space (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          docker system prune --all -f
-          Remove-Item "C:\Android" -Force -Recurse
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-          cache-directories: ${{ env.CARGO_TARGET_DIR }}
-
-      - name: "Check"
-        run: cargo check --all --exclude dora-dav1d --exclude dora-rav1e
-      - name: "Build  (Without Python dep as it is build with maturin)"
-        run: cargo build --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
-      - name: "Test"
-        run: cargo test --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
-
-  # Run examples as separate job because otherwise we will exhaust the disk
-  # space of the GitHub action runners.
-  examples:
-    name: "Examples"
-    strategy:
-      matrix:
-        platform: [ubuntu-22.04, macos-latest, windows-latest]
-      fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        if: runner.os == 'Linux'
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: true
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
-      - name: Free disk Space (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          docker system prune --all -f
-          Remove-Item "C:\Android" -Force -Recurse
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      # general examples
-      - name: "Build examples"
-        timeout-minutes: 30
-        run: cargo build --examples
-      - name: "Rust Dataflow example"
-        timeout-minutes: 30
-        run: cargo run --example rust-dataflow
-      - name: "Rust Git Dataflow example"
-        timeout-minutes: 30
-        run: cargo run --example rust-dataflow-git
-      - name: "Multiple Daemons example"
-        timeout-minutes: 30
-        run: cargo run --example multiple-daemons
-      - name: "C Dataflow example"
-        timeout-minutes: 15
-        run: cargo run --example c-dataflow
-      - name: "C++ Dataflow example"
-        timeout-minutes: 15
-        run: cargo run --example cxx-dataflow
-      - name: "Install Arrow C++ Library"
-        timeout-minutes: 10
-        shell: bash
-        run: |
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            # For Ubuntu
-            sudo apt-get update
-            sudo apt-get install -y -V ca-certificates lsb-release wget
-            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-            sudo apt-get update
-            sudo apt-get install -y -V libarrow-dev libarrow-glib-dev
-          elif [ "$RUNNER_OS" == "macOS" ]; then
-            # For macOS
-            brew update
-            brew install apache-arrow
-          fi
-      - name: "C++ Dataflow2 example"
-        timeout-minutes: 15
-        run: cargo run --example cxx-arrow-dataflow
-      - name: "Cmake example"
-        if: runner.os == 'Linux'
-        timeout-minutes: 30
-        run: cargo run --example cmake-dataflow
-      - name: "Unix Domain Socket example"
-        if: runner.os == 'Linux'
-        run: cargo run --example rust-dataflow -- dataflow_socket.yml
-
-  # ROS2 bridge examples
-  ros2-bridge-examples:
-    name: "ROS2 Bridge Examples"
-    runs-on: ubuntu-22.04
-    timeout-minutes: 45
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        if: runner.os == 'Linux'
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: false
-
-      - run: cargo --version --verbose
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - uses: ros-tooling/setup-ros@v0.7
-        with:
-          required-ros-distributions: humble
-      - run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"'
-      - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-      - name: Install pyarrow
-        run: pip install pyarrow
-      - name: "Test"
-        run: cargo test -p dora-ros2-bridge-python
-      - name: "Rust ROS2 Bridge example"
-        timeout-minutes: 30
-        env:
-          QT_QPA_PLATFORM: offscreen
-        run: |
-          source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node &
-          source /opt/ros/humble/setup.bash && ros2 run examples_rclcpp_minimal_service service_main &
-          cargo run --example rust-ros2-dataflow --features="ros2-examples"
-      - uses: actions/setup-python@v5
-        if: runner.os != 'Windows'
-        with:
-          python-version: "3.8"
-      - uses: actions/setup-python@v5
-        if: runner.os == 'Windows'
-        with:
-          python-version: "3.10"
-      - name: "python-ros2-dataflow"
-        timeout-minutes: 30
-        env:
-          QT_QPA_PLATFORM: offscreen
-        run: |
-          # Reset only the turtlesim instance as it is not destroyed at the end of the previous job
-          source /opt/ros/humble/setup.bash && ros2 service call /reset std_srvs/srv/Empty &
-          cargo run --example python-ros2-dataflow --features="ros2-examples"
-      - name: "c++-ros2-dataflow"
-        timeout-minutes: 30
-        env:
-          QT_QPA_PLATFORM: offscreen
-        run: |
-          # Reset only the turtlesim instance as it is not destroyed at the end of the previous job
-          source /opt/ros/humble/setup.bash && ros2 service call /reset std_srvs/srv/Empty &
-          cargo run --example cxx-ros2-dataflow --features="ros2-examples"
-
-  bench:
-    name: "Bench"
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-      fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: "Benchmark example"
-        timeout-minutes: 30
-        run: cargo run --example benchmark --release
-
-  CLI:
-    name: "CLI Test"
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-      fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        if: runner.os == 'Linux'
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: true
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      # CLI tests
-      - name: "Build cli and binaries"
-        timeout-minutes: 45
-        # fail-fast by using bash shell explictly
-        shell: bash
-        run: |
-          cargo install --path binaries/cli --locked
-      - name: "Test CLI (Rust)"
-        timeout-minutes: 45
-        # fail-fast by using bash shell explictly
-        shell: bash
-        run: |
-          # Test Rust template Project
-          dora new test_rust_project --internal-create-with-path-dependencies
-          cd test_rust_project
-          cargo build --all --exclude dora-dav1d --exclude dora-rav1e
-          dora up
-          dora list
-          dora start dataflow.yml --name ci-rust-test --detach
-          sleep 10
-          dora stop --name ci-rust-test  --grace-duration 5s
-          cd ..
-          dora build examples/rust-dataflow/dataflow_dynamic.yml
-          dora start examples/rust-dataflow/dataflow_dynamic.yml --name ci-rust-dynamic --detach
-          cargo run -p rust-dataflow-example-sink-dynamic
-          sleep 5
-          dora stop --name ci-rust-dynamic  --grace-duration 5s
-          dora destroy
-
-      - uses: actions/setup-python@v5
-        with:
-          # TODO: Support Python 3.13 when https://github.com/pytorch/pytorch/issues/130249 is fixed
-          python-version: "3.12"
-
-      - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v5
-        with:
-          enable-cache: true
-
-      - name: "Test CLI (Python)"
-        timeout-minutes: 45
-        # fail-fast by using bash shell explictly
-        shell: bash
-        run: |
-          # Test Python template Project
-          dora new test_python_project --lang python --internal-create-with-path-dependencies
-          cd test_python_project
-          uv venv --seed -p 3.12
-          uv pip install -e ../apis/python/node
-          uv pip install ruff pytest
-
-          echo "Running dora up"
-          dora up
-          echo "Running dora build"
-          dora build dataflow.yml --uv
-
-          # Check Compliancy
-          uv run ruff check .
-          uv run pytest
-
-          export OPERATING_MODE=SAVE
-          echo "Running dora list"
-          dora list
-          dora build dataflow.yml --uv
-          echo "Running CI Python Test"
-          dora start dataflow.yml --name ci-python-test --detach --uv
-          sleep 10
-          echo "Running dora stop"
-          dora stop --name ci-python-test  --grace-duration 5s
-          dora destroy
-          sleep 5
-
-          cd ..
-
-          # Run Python Node Example
-          echo "Running Python Node Example"
-          dora up
-          uv venv --seed -p 3.12
-          uv pip install -e apis/python/node
-          dora build examples/python-dataflow/dataflow.yml --uv
-          dora start examples/python-dataflow/dataflow.yml --name ci-python --detach --uv
-          sleep 10
-          echo "Running dora stop"
-          dora stop --name ci-python  --grace-duration 30s
-
-          # Run Python Dynamic Node Example
-          echo "Running Python Node Dynamic Example"
-          dora build examples/python-dataflow/dataflow_dynamic.yml --uv
-          dora start examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach --uv
-          uv run opencv-plot --name plot
-          sleep 10
-          echo "Running dora stop"
-          dora stop --name ci-python-dynamic  --grace-duration 30s
-
-          # Run Python Operator Example
-          echo "Running CI Operator Test"
-          dora start examples/python-operator-dataflow/dataflow.yml --name ci-python-operator --detach --uv
-          sleep 10
-          echo "Running dora stop"
-          dora stop --name ci-python-operator  --grace-duration 30s
-
-          dora destroy
-          sleep 5
-
-          # Run Python queue latency test
-          echo "Running CI Queue Latency Test"
-          dora run tests/queue_size_latest_data_python/dataflow.yaml --uv
-
-          # Run Python queue latency test + timeout
-          echo "Running CI Queue + Timeout Test"
-          dora run tests/queue_size_and_timeout_python/dataflow.yaml --uv
-
-          # Run Rust queue latency test
-          echo "Running CI Queue Size Latest Data Rust Test"
-          dora build tests/queue_size_latest_data_rust/dataflow.yaml --uv
-          dora run tests/queue_size_latest_data_rust/dataflow.yaml --uv
-
-      - name: "Test CLI (C)"
-        timeout-minutes: 45
-        # fail-fast by using bash shell explictly
-        shell: bash
-        if: runner.os == 'Linux'
-        run: |
-          # Test C template Project
-          dora new test_c_project --lang c --internal-create-with-path-dependencies
-          cd test_c_project
-          dora up
-          dora list
-          cmake -B build
-          cmake --build build
-          cmake --install build
-          dora start dataflow.yml --name ci-c-test --detach
-          sleep 10
-          dora stop --name ci-c-test  --grace-duration 5s
-          dora destroy
-
-      - name: "Test CLI (C++)"
-        timeout-minutes: 45
-        # fail-fast by using bash shell explictly
-        shell: bash
-        if: runner.os == 'Linux'
-        run: |
-          # Test C++ template Project
-          dora new test_cxx_project --lang cxx --internal-create-with-path-dependencies
-          cd test_cxx_project
-          dora up
-          dora list
-          cmake -B build
-          cmake --build build
-          cmake --install build
-          dora start dataflow.yml --name ci-cxx-test --detach
-          sleep 10
-          dora stop --name ci-cxx-test  --grace-duration 5s
-          dora destroy
-
-  clippy:
-    name: "Clippy"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-
-      - name: "Clippy"
-        run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e
-      - name: "Clippy (tracing feature)"
-        run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features tracing
-        if: false # only the dora-runtime has this feature, but it is currently commented out
-      - name: "Clippy (metrics feature)"
-        run: cargo clippy --all --exclude dora-dav1d --exclude dora-rav1e --features metrics
-        if: false # only the dora-runtime has this feature, but it is currently commented out
-
-  rustfmt:
-    name: "Formatting"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - name: "rustfmt"
-        run: cargo fmt --all -- --check
-
-  check-license:
-    name: "License Checks"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          cache-on-failure: true
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - run: cargo install cargo-lichking
-      - name: "Check dependency licenses"
-        run: cargo lichking check
-
-  typos:
-    name: Typos
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - name: Typos check with custom config file
-        uses: crate-ci/typos@master
-
-  cross-check:
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      matrix:
-        platform:
-          - runner: ubuntu-22.04
-            target: x86_64-unknown-linux-gnu
-          - runner: ubuntu-22.04
-            target: i686-unknown-linux-gnu
-          - runner: ubuntu-22.04
-            target: aarch64-unknown-linux-gnu
-          - runner: ubuntu-22.04
-            target: aarch64-unknown-linux-musl
-          - runner: ubuntu-22.04
-            target: armv7-unknown-linux-musleabihf
-          - runner: ubuntu-22.04
-            target: x86_64-pc-windows-gnu
-          - runner: macos-13
-            target: aarch64-apple-darwin
-          - runner: macos-13
-            target: x86_64-apple-darwin
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - name: "Add toolchains"
-        run: rustup target add ${{ matrix.platform.target }}
-      - name: Install system-level dependencies
-        if: runner.target == 'x86_64-pc-windows-gnu'
-        run: |
-          sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
-      - name: "Check"
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: check
-          args: --target  ${{ matrix.platform.target }} --all --exclude dora-dav1d --exclude dora-rav1e --exclude dora-node-api-python --exclude dora-operator-api-python --exclude dora-ros2-bridge-python
-
-  # copied from https://github.com/rust-lang/cargo/blob/6833aa715d724437dc1247d0166afe314ab6854e/.github/workflows/main.yml#L291
-  msrv:
-    name: "Check for specified `rust-version`"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: taiki-e/install-action@cargo-hack
-    - run: cargo hack check --all-targets --rust-version --workspace --ignore-private --locked --exclude dora-rav1e --exclude dora-dav1d

.github/workflows/delete-buildjet-cache.yml

@@ -1,18 +0,0 @@
-name: Manually Delete BuildJet Cache
-on:
-  workflow_dispatch:
-    inputs:
-      cache_key:
-        description: 'BuildJet Cache Key to Delete'
-        required: true
-        type: string
-
-jobs:
-  manually-delete-buildjet-cache:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: buildjet/cache-delete@v1
-        with:
-          cache_key: ${{ inputs.cache_key }}

.github/workflows/docker-image.yml

@@ -1,31 +0,0 @@
-name: Docker Image CI/CD
-
-on:
-  push:
-    branches: ["main"]
-    paths:
-      - "docker/**"
-  pull_request:
-    paths:
-      - "docker/**"
-
-jobs:
-  build_and_push:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: "Login to GitHub Container Registry"
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{github.actor}}
-          password: ${{secrets.GITHUB_TOKEN}}
-
-      - name: Build the Docker image
-        run: |
-          docker build docker/slim --platform linux/amd64,linux/arm64,linux/arm32v6,linux/arm32v7 -t multi-platform --tag ghcr.io/dora-rs/dora-slim:latest
-          docker push ghcr.io/dora-rs/dora-slim:latest

.github/workflows/dora-bot-assign.yml

@@ -1,130 +0,0 @@
-name: "Dora Bot"
-
-on:
-  issue_comment:
-    types: [created]
-  schedule:
-    - cron: "0 0 * * *" # Midnight(UTC)
-
-jobs:
-  assign-unassign:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-    if: github.event_name == 'issue_comment'
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Parses comment then assign/unassign user
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_BODY: "${{ github.event.comment.body }}"
-          ISSUE_NUMBER: "${{ github.event.issue.number }}"
-          COMMENT_AUTHOR: "${{ github.event.comment.user.login }}"
-          AUTHOR_ASSOCIATION: "${{ github.event.comment.author_association }}"
-        run: |
-          # For assigning
-          if [[ "$COMMENT_BODY" == "@dora-bot assign me" ]]; then
-            echo "Assigning $COMMENT_AUTHOR to issue #$ISSUE_NUMBER"
-            curl -X POST \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/assignees \
-              -d "{\"assignees\":[\"$COMMENT_AUTHOR\"]}"
-
-            # Returns a comment back 
-            curl -X POST \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/comments \
-              -d "{\"body\":\"Hello @$COMMENT_AUTHOR, this issue is now assigned to you!\"}"
-
-          # for unassigning(self)
-          elif [[ "$COMMENT_BODY" == "@dora-bot unassign me" ]]; then
-            echo "Unassigning $COMMENT_AUTHOR from issue #$ISSUE_NUMBER"
-            curl -X DELETE \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/assignees \
-              -d "{\"assignees\":[\"$COMMENT_AUTHOR\"]}"
-
-            # Returns a comment back
-            curl -X POST \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/comments \
-              -d "{\"body\":\"Hello @$COMMENT_AUTHOR, you have been unassigned from this issue.\"}"
-
-          # Command to help maintainers to unassign 
-          elif [[ "$COMMENT_BODY" =~ @dora-bot\ unassign\ [@]?([a-zA-Z0-9_-]+) ]]; then
-            TARGET_USER="${BASH_REMATCH[1]}"
-
-            # Checking that the comment author has proper permissions
-            if [[ "$AUTHOR_ASSOCIATION" == "NONE" || "$AUTHOR_ASSOCIATION" == "CONTRIBUTOR" ]]; then
-              echo "Unauthorized unassign command by $COMMENT_AUTHOR. Only maintainers or collaborators may unassign others."
-              exit 1
-            fi
-
-            echo "Maintainer $COMMENT_AUTHOR is unassigning $TARGET_USER from issue #$ISSUE_NUMBER"
-            curl -X DELETE \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/assignees \
-              -d "{\"assignees\":[\"$TARGET_USER\"]}"
-
-            curl -X POST \
-              -H "Authorization: Bearer $GITHUB_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              https://api.github.com/repos/${{ github.repository }}/issues/$ISSUE_NUMBER/comments \
-              -d "{\"body\":\"Hello @$TARGET_USER, you have been unassigned from this issue by @$COMMENT_AUTHOR.\"}"
-          else
-            echo "No matching command found in comment: $COMMENT_BODY"
-          fi
-
-  stale-unassign:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-    if: github.event_name == 'schedule'
-    steps:
-      - name: Unassign stale issues
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Calculate the timestamp for 14 days ago
-          TWO_WEEKS_AGO=$(date -d "14 days ago" +%s)
-          repo="${{ github.repository }}"
-          echo "Fetching open issues for $repo"
-          issues=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
-                      -H "Accept: application/vnd.github+json" \
-                      "https://api.github.com/repos/${repo}/issues?state=open&per_page=100")
-          
-          issue_count=$(echo "$issues" | jq '. | length')
-          echo "Found $issue_count open issues"
-          
-          for (( i=0; i<$issue_count; i++ )); do
-            issue_number=$(echo "$issues" | jq -r ".[$i].number")
-            updated_at=$(echo "$issues" | jq -r ".[$i].updated_at")
-            updated_ts=$(date -d "$updated_at" +%s)
-            
-            # If the issue hasn't been updated within 2 weeks, consider it stale.
-            if [[ $updated_ts -lt $TWO_WEEKS_AGO ]]; then
-              assignees=$(echo "$issues" | jq -r ".[$i].assignees | .[].login")
-              if [[ -n "$assignees" ]]; then
-                echo "Issue #$issue_number is stale. Unassigning users: $assignees"
-                for user in $assignees; do
-                  curl -X DELETE \
-                    -H "Authorization: Bearer $GITHUB_TOKEN" \
-                    -H "Accept: application/vnd.github+json" \
-                    https://api.github.com/repos/${repo}/issues/$issue_number/assignees \
-                    -d "{\"assignees\":[\"$user\"]}"
-                  curl -X POST \
-                    -H "Authorization: Bearer $GITHUB_TOKEN" \
-                    -H "Accept: application/vnd.github+json" \
-                    https://api.github.com/repos/${repo}/issues/$issue_number/comments \
-                    -d "{\"body\":\"@${user} has been automatically unassigned from this stale issue after 2 weeks of inactivity.\"}"
-                done
-              fi
-            fi
-          done

.github/workflows/labeler.yml

@@ -1,17 +0,0 @@
-name: "Issue Labeler"
-on:
-  issues:
-    types: [opened, edited]
-
-jobs:
-  triage:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-    steps:
-      - uses: github/issue-labeler@v3.1 #May not be the latest version
-        with:
-          repo-token: "${{ github.token }}"
-          configuration-path: .github/labeler.yml
-          enable-versioned-regex: 0
-          include-title: 1

.github/workflows/node-hub-ci-cd.yml

@@ -1,96 +0,0 @@
-name: node-hub
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-  release:
-    types: [published]
-
-jobs:
-  find-jobs:
-    runs-on: ubuntu-24.04
-    name: Find Jobs
-    outputs:
-      folders: ${{ steps.jobs.outputs.folders }}
-    steps:
-      - uses: actions/checkout@v1
-
-      - id: jobs
-        uses: kmanimaran/list-folder-action@v4
-        with:
-          path: ./node-hub
-
-  ci:
-    runs-on: ${{ matrix.platform }}
-    needs: [find-jobs]
-    defaults:
-      run:
-        working-directory: node-hub/${{ matrix.folder }}
-    strategy:
-      fail-fast: ${{ github.event_name != 'workflow_dispatch' && !(github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) }}
-      matrix:
-        platform: [ubuntu-24.04, macos-14]
-        folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
-    steps:
-      - name: Checkout repository
-        if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
-        uses: actions/checkout@v4
-        with:
-          submodules: true # Make sure to check out the sub-module
-
-      - name: Update submodule
-        if: runner.os == 'Linux'
-        run: |
-          git submodule update --init --recursive
-          git submodule update --remote --recursive
-
-      - name: Install system-level dependencies
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt update
-          sudo apt-get install portaudio19-dev
-          sudo apt-get install libdav1d-dev nasm libudev-dev
-          mkdir -p $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib
-          ln -s /lib/x86_64-linux-gnu/libdav1d.so $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/libdav1d.so
-
-          # Install mingw-w64 cross-compilers
-          sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
-
-      - name: Install system-level dependencies for MacOS
-        if: runner.os == 'MacOS' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')))
-        run: |
-          brew install portaudio
-          brew install dav1d nasm
-
-      - name: Set up Python
-        if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.10"
-
-      - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v5
-
-      - name: Set up Rust
-        if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-
-      - name: Run Linting and Tests
-        ## Run Linting and testing only on Mac for release workflows.
-        if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }}
-          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASS }}
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: |
-          chmod +x ../../.github/workflows/node_hub_test.sh
-          ../../.github/workflows/node_hub_test.sh

.github/workflows/node_hub_test.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-set -euo
-
-# Check if we are running in a GitHub Actions environment
-CI=${GITHUB_ACTIONS:-false}
-
-# List of ignored modules 
-ignored_folders=("dora-parler" "dora-opus" "dora-internvl" "dora-magma")
-
-# Skip test
-skip_test_folders=("dora-internvl" "dora-parler" "dora-keyboard" "dora-microphone" "terminal-input" "dora-magma")
-
-# Get current working directory
-dir=$(pwd)
-
-# Get the base name of the directory (without the path)
-base_dir=$(basename "$dir")
-
-export GIT_LFS_SKIP_SMUDGE=1 
-# Large node list requiring space cleanup
-large_node=("dora-phi4")
-
-export PYTEST_ADDOPTS="-x"
-
-# Check if the current directory is in the large node list and if we're in the CI environment
-if [[ " ${large_node[@]} " =~ " ${base_dir} " ]] && [[ "$CI" == "true" ]]; then
-    echo "Running cleanup for $base_dir..."
-    sudo rm -rf /opt/hostedtoolcache/CodeQL || :
-    # 1.4GB
-    sudo rm -rf /opt/hostedtoolcache/go || :
-    # 489MB
-    sudo rm -rf /opt/hostedtoolcache/PyPy || :
-    # 376MB
-    sudo rm -rf /opt/hostedtoolcache/node || :
-    # Remove Web browser packages
-    sudo apt purge -y \
-    firefox \
-    google-chrome-stable \
-    microsoft-edge-stable
-    sudo rm -rf /usr/local/lib/android/ 
-    sudo rm -rf /usr/share/dotnet/
-    sudo rm -rf /opt/ghc/
-fi
-
-# Check if the directory name is in the ignored list
-if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
-    echo "Skipping $base_dir as we cannot test it on the CI..."
-else
-    # IF job is mixed rust-python job and is on linux 
-    if [[ -f "Cargo.toml" && -f "pyproject.toml" &&  "$(uname)" = "Linux" ]]; then
-        echo "Running build and tests for Rust project in $dir..."
-
-        cargo check
-        cargo clippy
-        cargo build
-        cargo test
-
-        pip install "maturin[zig, patchelf]"
-        maturin build --release --compatibility manylinux_2_28 --zig
-        # If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel on multiple platforms
-        if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
-            # Free up ubuntu space
-            sudo apt-get clean
-            sudo rm -rf /usr/local/lib/android/ 
-            sudo rm -rf /usr/share/dotnet/
-            sudo rm -rf /opt/ghc/
-
-            maturin publish --skip-existing --compatibility manylinux_2_28 --zig
-            # aarch64-unknown-linux-gnu
-            rustup target add aarch64-unknown-linux-gnu
-            maturin publish --target aarch64-unknown-linux-gnu --skip-existing --zig  --compatibility manylinux_2_28
-                
-            # armv7-unknown-linux-musleabihf
-            rustup target add armv7-unknown-linux-musleabihf
-            # If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
-            maturin publish --target armv7-unknown-linux-musleabihf --skip-existing --zig
-
-            # x86_64-pc-windows-gnu
-            rustup target add x86_64-pc-windows-gnu
-            # If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
-            maturin publish --target x86_64-pc-windows-gnu --skip-existing 
-        fi
-
-    elif [[ -f "Cargo.toml" && -f "pyproject.toml" &&  "$(uname)" = "Darwin" ]]; then
-        pip install "maturin[zig, patchelf]"
-        # aarch64-apple-darwin
-        maturin build  --release
-        # If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
-        if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
-            maturin publish --skip-existing
-        fi
-
-    elif [[ "$(uname)" = "Linux" ]] || [[ "$CI" == "false" ]]; then
-        if [ -f "$dir/Cargo.toml" ]; then
-            echo "Running build and tests for Rust project in $dir..."
-            cargo check
-            cargo clippy
-            cargo build
-            cargo test
-
-            if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
-                cargo publish
-            fi
-        else
-            if [ -f "$dir/pyproject.toml" ]; then
-            echo "CI: Installing in $dir..."
-            uv venv --seed -p 3.11
-            uv pip install .
-            echo "CI: Running Linting in $dir..."
-            uv run ruff check .
-            echo "CI: Running Pytest in $dir..."
-            # Skip test for some folders
-            if [[ " ${skip_test_folders[@]} " =~ " ${base_dir} " ]]; then
-                echo "Skipping tests for $base_dir..."
-            else
-                uv run pytest
-            fi
-            if [ "${GITHUB_EVENT_NAME:-false}" == "release" ] || [ "${GITHUB_EVENT_NAME:-false}" == "workflow_dispatch" ]; then
-                uv build
-                uv publish --check-url https://pypi.org/simple
-            fi
-            fi
-        fi 
-    fi 
-fi

.github/workflows/pip-release.yml

@@ -1,266 +0,0 @@
-# This file has been originally generated by maturin v0.14.17
-# To update, you can check
-#
-#    maturin generate-ci github --zig
-#
-# But note that some manual modification has been done.
-# Check the diffs to make sure that you haven't broken anything.
-
-name: pip-release
-
-on:
-  release:
-    types:
-      - "published"
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - "apis/python/node/**"
-      - "binaries/cli/**"
-      - "Cargo.toml"
-
-permissions:
-  contents: write
-
-jobs:
-  linux:
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - runner: ubuntu-22.04
-            target: x86_64
-          - runner: ubuntu-22.04
-            target: x86
-          - runner: ubuntu-22.04
-            target: aarch64
-          - runner: ubuntu-22.04
-            target: armv7
-          # - runner: ubuntu-22.04
-          # target: s390x
-          # - runner: ubuntu-22.04
-          # target: ppc64le
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-provider: buildjet
-          # only save caches for `main` branch
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist --zig
-          manylinux: manylinux_2_28
-          working-directory: ${{ matrix.repository.path }}
-          before-script-linux: sudo apt-get install libatomic1-i386-cross libatomic1-armhf-cross && mkdir -p $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/ && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so.1 && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 /opt/hostedtoolcache/Python/3.8.18/x64/lib/libatomic.so.1 && mkdir -p $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/ && ln -s /usr/arm-linux-gnueabihf/lib/libatomic.so.1 $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/libatomic.so
-      - name: Upload wheels
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-linux-${{ matrix.platform.target }}
-          path: ${{ matrix.repository.path }}/dist
-
-  musllinux:
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - runner: ubuntu-22.04
-            target: x86_64
-          - runner: ubuntu-22.04
-            target: x86
-          - runner: ubuntu-22.04
-            target: aarch64
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist
-          sccache: "false"
-          manylinux: musllinux_1_2
-          working-directory: ${{ matrix.repository.path }}
-
-      - name: Upload wheels
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-musllinux-${{ matrix.platform.target }}
-          path: ${{ matrix.repository.path }}/dist
-
-  musleabi:
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          [
-            {
-              target: "armv7-unknown-linux-musleabihf",
-              image_tag: "armv7-musleabihf",
-            },
-          ]
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    container:
-      image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }}
-      env:
-        CFLAGS_armv7_unknown_linux_musleabihf: "-mfpu=vfpv3-d16"
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - name: Build Wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          manylinux: auto
-          container: off
-          args: --release -o dist
-          working-directory: ${{ matrix.repository.path }}
-      - name: Upload wheels
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-musllinux-${{ matrix.platform.target }}
-          path: ${{ matrix.repository.path }}/dist
-
-  windows:
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - runner: windows-latest
-            target: x64
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-          architecture: ${{ matrix.platform.target }}
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.8
-          sccache: "true"
-          working-directory: ${{ matrix.repository.path }}
-      - name: Upload wheels
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-windows-${{ matrix.platform.target }}
-          path: ${{ matrix.repository.path }}/dist
-
-  macos:
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - runner: macos-13
-            target: aarch64
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.8
-          working-directory: ${{ matrix.repository.path }}
-      - name: Upload wheels
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-macos-${{ matrix.platform.target }}
-          path: ${{ matrix.repository.path }}/dist
-
-  sdist:
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build sdist
-        uses: PyO3/maturin-action@v1
-        with:
-          command: sdist
-          args: --out dist
-          working-directory: ${{ matrix.repository.path }}
-      - name: Upload sdist
-        if: github.event_name == 'release'
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.repository.name }}-sdist
-          path: ${{ matrix.repository.path }}/dist
-
-  release:
-    name: Release
-    runs-on: ubuntu-22.04
-    if: github.event_name == 'workflow_dispatch' || github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')
-    needs: [linux, musllinux, musleabi, windows, macos, sdist]
-    strategy:
-      fail-fast: false
-      matrix:
-        repository:
-          - path: apis/python/node
-            name: dora-node-api
-          - path: binaries/cli
-            name: dora-rs-cli
-    steps:
-      - uses: actions/download-artifact@v4
-      - name: Publish to PyPI
-        uses: PyO3/maturin-action@v1
-        env:
-          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }}
-        with:
-          command: upload
-          args: --non-interactive --skip-existing ${{ matrix.repository.name }}-*/*

.github/workflows/regenerate-schemas.yml

@@ -1,40 +0,0 @@
-name: Regenerate JSON schemas
-
-on:
-  push:
-    branches: ["main"]
-
-jobs:
-  regenerate_schemas:
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-      - uses: r7kamura/rust-problem-matchers@v1.1.0
-      - run: cargo --version --verbose
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-on-failure: true
-
-      - name: Update Schema
-        run: cargo run -p dora-core --bin generate_schema
-      - name: Create if changed
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          if git diff --exit-code -- libraries/core/dora-schema.json; then
-              echo "Schema file was not changed"
-          else
-              git switch -c schema-update-for-${{ github.sha }}
-              git add libraries/core/dora-schema.json
-              git config user.email "dora-bot@phil-opp.com"
-              git config user.name "Dora Bot"
-              git commit -m "Update JSON schema for ${{ github.sha }}"
-              git push -u origin HEAD
-              git fetch origin main
-              gh pr create --title "Update JSON schema for \`dora-core\`" --body "Update JSON schema for ${{ github.sha }}"
-          fi
-          

.github/workflows/release.yml

@@ -1,289 +0,0 @@
-# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
-#
-# Copyright 2022-2024, axodotdev
-# SPDX-License-Identifier: MIT or Apache-2.0
-#
-# CI that:
-#
-# * checks for a Git Tag that looks like a release
-# * builds artifacts with dist (archives, installers, hashes)
-# * uploads those artifacts to temporary workflow zip
-# * on success, uploads the artifacts to a GitHub Release
-#
-# Note that a GitHub Release with this tag is assumed to exist as a draft
-# with the appropriate title/body, and will be undrafted for you.
-
-name: Release
-permissions:
-  "contents": "write"
-
-# This task will run whenever you push a git tag that looks like a version
-# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
-# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
-# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
-# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
-#
-# If PACKAGE_NAME is specified, then the announcement will be for that
-# package (erroring out if it doesn't have the given version or isn't dist-able).
-#
-# If PACKAGE_NAME isn't specified, then the announcement will be for all
-# (dist-able) packages in the workspace with that version (this mode is
-# intended for workspaces with only one dist-able package, or with all dist-able
-# packages versioned/released in lockstep).
-#
-# If you push multiple tags at once, separate instances of this workflow will
-# spin up, creating an independent announcement for each one. However, GitHub
-# will hard limit this to 3 tags per commit, as it will assume more tags is a
-# mistake.
-#
-# If there's a prerelease-style suffix to the version, then the release(s)
-# will be marked as a prerelease.
-on:
-  release:
-    types:
-      - "published"
-  workflow_dispatch:
-
-jobs:
-  # Run 'dist plan' (or host) to determine what tasks we need to do
-  plan:
-    runs-on: "ubuntu-22.04"
-    outputs:
-      val: ${{ steps.plan.outputs.manifest }}
-      tag: ${{ !github.event.pull_request && github.ref_name || '' }}
-      tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
-      publishing: ${{ !github.event.pull_request }}
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: Install dist
-        # we specify bash to get pipefail; it guards against the `curl` command
-        # failing. otherwise `sh` won't catch that `curl` returned non-0
-        shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
-      - name: Cache dist
-        uses: actions/upload-artifact@v4
-        with:
-          name: cargo-dist-cache
-          path: ~/.cargo/bin/dist
-      # sure would be cool if github gave us proper conditionals...
-      # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
-      # functionality based on whether this is a pull_request, and whether it's from a fork.
-      # (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
-      # but also really annoying to build CI around when it needs secrets to work right.)
-      - id: plan
-        run: |
-          dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
-          echo "dist ran successfully"
-          cat plan-dist-manifest.json
-          echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
-      - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifacts-plan-dist-manifest
-          path: plan-dist-manifest.json
-
-  # Build and packages all the platform-specific things
-  build-local-artifacts:
-    name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
-    # Let the initial task tell us to not run (currently very blunt)
-    needs:
-      - plan
-    if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
-    strategy:
-      fail-fast: false
-      # Target platforms/runners are computed by dist in create-release.
-      # Each member of the matrix has the following arguments:
-      #
-      # - runner: the github runner
-      # - dist-args: cli flags to pass to dist
-      # - install-dist: expression to run to install dist on the runner
-      #
-      # Typically there will be:
-      # - 1 "global" task that builds universal installers
-      # - N "local" tasks that build each platform's binaries and platform-specific installers
-      matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
-    runs-on: ${{ matrix.runner }}
-    container: ${{ matrix.container && matrix.container.image || null }}
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
-    steps:
-      - name: enable windows longpaths
-        run: |
-          git config --global core.longpaths true
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: Install Rust non-interactively if not already installed
-        if: ${{ matrix.container }}
-        run: |
-          if ! command -v cargo > /dev/null 2>&1; then
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-          fi
-      - name: Install dist
-        run: ${{ matrix.install_dist.run }}
-      # Get the dist-manifest
-      - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts-*
-          path: target/distrib/
-          merge-multiple: true
-      - name: Install dependencies
-        run: |
-          ${{ matrix.packages_install }}
-      - name: Build artifacts
-        run: |
-          # Actually do builds and make zips and whatnot
-          dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "dist ran successfully"
-      - id: cargo-dist
-        name: Post-build
-        # We force bash here just because github makes it really hard to get values up
-        # to "real" actions without writing to env-vars, and writing to env-vars has
-        # inconsistent syntax between shell and powershell.
-        shell: bash
-        run: |
-          # Parse out what we just built and upload it to scratch storage
-          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
-          echo "EOF" >> "$GITHUB_OUTPUT"
-
-          cp dist-manifest.json "$BUILD_MANIFEST_NAME"
-      - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifacts-build-local-${{ join(matrix.targets, '_') }}
-          path: |
-            ${{ steps.cargo-dist.outputs.paths }}
-            ${{ env.BUILD_MANIFEST_NAME }}
-
-  # Build and package all the platform-agnostic(ish) things
-  build-global-artifacts:
-    needs:
-      - plan
-      - build-local-artifacts
-    runs-on: "ubuntu-22.04"
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: Install cached dist
-        uses: actions/download-artifact@v4
-        with:
-          name: cargo-dist-cache
-          path: ~/.cargo/bin/
-      - run: chmod +x ~/.cargo/bin/dist
-      # Get all the local artifacts for the global tasks to use (for e.g. checksums)
-      - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts-*
-          path: target/distrib/
-          merge-multiple: true
-      - id: cargo-dist
-        shell: bash
-        run: |
-          dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
-          echo "dist ran successfully"
-
-          # Parse out what we just built and upload it to scratch storage
-          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
-          echo "EOF" >> "$GITHUB_OUTPUT"
-
-          cp dist-manifest.json "$BUILD_MANIFEST_NAME"
-      - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifacts-build-global
-          path: |
-            ${{ steps.cargo-dist.outputs.paths }}
-            ${{ env.BUILD_MANIFEST_NAME }}
-  # Determines if we should publish/announce
-  host:
-    needs:
-      - plan
-      - build-local-artifacts
-      - build-global-artifacts
-    # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
-    if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    runs-on: "ubuntu-22.04"
-    outputs:
-      val: ${{ steps.host.outputs.manifest }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: Install cached dist
-        uses: actions/download-artifact@v4
-        with:
-          name: cargo-dist-cache
-          path: ~/.cargo/bin/
-      - run: chmod +x ~/.cargo/bin/dist
-      # Fetch artifacts from scratch-storage
-      - name: Fetch artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts-*
-          path: target/distrib/
-          merge-multiple: true
-      - id: host
-        shell: bash
-        run: |
-          dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
-          echo "artifacts uploaded and released successfully"
-          cat dist-manifest.json
-          echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
-      - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
-        with:
-          # Overwrite the previous copy
-          name: artifacts-dist-manifest
-          path: dist-manifest.json
-      # Create a GitHub Release while uploading all files to it
-      - name: "Download GitHub Artifacts"
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts-*
-          path: artifacts
-          merge-multiple: true
-      - name: Cleanup
-        run: |
-          # Remove the granular manifests
-          rm -f artifacts/*-dist-manifest.json
-      - name: Create GitHub Release
-        env:
-          PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
-          RELEASE_COMMIT: "${{ github.sha }}"
-        run: |
-          # If we're editing a release in place, we need to upload things ahead of time
-          gh release upload "${{ needs.plan.outputs.tag }}" artifacts/*
-
-          gh release edit "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --draft=false
-
-  announce:
-    needs:
-      - plan
-      - host
-    # use "always() && ..." to allow us to wait for all publish jobs while
-    # still allowing individual publish jobs to skip themselves (for prereleases).
-    # "host" however must run to completion, no skipping allowed!
-    if: ${{ always() && needs.host.result == 'success' }}
-    runs-on: "ubuntu-22.04"
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive

.gitignore

@@ -1,183 +0,0 @@
-# Generated by Cargo
-# will have compiled files and executables
-/target/
-examples/**/*.txt
-# These are backup files generated by rustfmt
-**/*.rs.bk
-
-# Remove arrow file from dora-record
-**/*.arrow
-*.pt
-
-# Remove hdf and stl files
-*.stl
-*.dae
-*.STL
-*.hdf5
-
-# Removing images.
-*.jpg
-*.mp3
-*.wav
-*.png
-!docs/src/latency.png
-
-# Remove *.html. Likely be outputed mermaid graph.
-*.html
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-/build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-.ruff_cache
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# log output file
-out/
-
-# MacOS DS_Store
-.DS_Store
-
-#Miscellaneous
-yolo.yml
-
-~*

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "node-hub/dora-rdt-1b/dora_rdt_1b/RoboticsDiffusionTransformer"]
-	path = node-hub/dora-rdt-1b/dora_rdt_1b/RoboticsDiffusionTransformer
-	url = https://github.com/thu-ml/RoboticsDiffusionTransformer
-[submodule "node-hub/dora-magma/dora_magma/Magma"]
-	path = node-hub/dora-magma/dora_magma/Magma
-	url = https://github.com/microsoft/Magma

.nojekyll

@@ -0,0 +1 @@
+This file makes sure that Github Pages doesn't process mdBook's output.

404.html

@@ -0,0 +1,187 @@
+<!DOCTYPE HTML>
+<html lang="en" class="sidebar-visible no-js light">
+    <head>
+        <!-- Book generated using mdBook -->
+        <meta charset="UTF-8">
+        <title></title>
+                        <base href="/">
+        
+
+        <!-- Custom HTML head -->
+        
+
+        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+        <meta name="description" content="">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+        <meta name="theme-color" content="#ffffff" />
+
+                <link rel="icon" href="favicon.svg">
+                        <link rel="shortcut icon" href="favicon.png">
+                <link rel="stylesheet" href="css/variables.css">
+        <link rel="stylesheet" href="css/general.css">
+        <link rel="stylesheet" href="css/chrome.css">
+                <link rel="stylesheet" href="css/print.css" media="print">
+        
+        <!-- Fonts -->
+        <link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
+                <link rel="stylesheet" href="fonts/fonts.css">
+        
+        <!-- Highlight.js Stylesheets -->
+        <link rel="stylesheet" href="highlight.css">
+        <link rel="stylesheet" href="tomorrow-night.css">
+        <link rel="stylesheet" href="ayu-highlight.css">
+
+        <!-- Custom theme stylesheets -->
+        
+            </head>
+    <body>
+        <!-- Provide site root to javascript -->
+        <script type="text/javascript">
+            var path_to_root = "";
+            var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
+        </script>
+
+        <!-- Work around some values being stored in localStorage wrapped in quotes -->
+        <script type="text/javascript">
+            try {
+                var theme = localStorage.getItem('mdbook-theme');
+                var sidebar = localStorage.getItem('mdbook-sidebar');
+
+                if (theme.startsWith('"') && theme.endsWith('"')) {
+                    localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
+                }
+
+                if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
+                    localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
+                }
+            } catch (e) { }
+        </script>
+
+        <!-- Set the theme before any content is loaded, prevents flash -->
+        <script type="text/javascript">
+            var theme;
+            try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
+            if (theme === null || theme === undefined) { theme = default_theme; }
+            var html = document.querySelector('html');
+            html.classList.remove('no-js')
+            html.classList.remove('light')
+            html.classList.add(theme);
+            html.classList.add('js');
+        </script>
+
+        <!-- Hide / unhide sidebar before it is displayed -->
+        <script type="text/javascript">
+            var html = document.querySelector('html');
+            var sidebar = 'hidden';
+            if (document.body.clientWidth >= 1080) {
+                try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
+                sidebar = sidebar || 'visible';
+            }
+            html.classList.remove('sidebar-visible');
+            html.classList.add("sidebar-" + sidebar);
+        </script>
+
+        <nav id="sidebar" class="sidebar" aria-label="Table of contents">
+            <div class="sidebar-scrollbox">
+                <ol class="chapter"><li class="chapter-item expanded affix "><a href="introduction.html">Introduction</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">User Guide</li><li class="chapter-item expanded "><a href="installation.html"><strong aria-hidden="true">1.</strong> installation</a></li><li class="chapter-item expanded "><a href="getting-started.html"><strong aria-hidden="true">2.</strong> Getting started</a></li><li class="chapter-item expanded affix "><li class="part-title">Reference Guide</li><li class="chapter-item expanded "><a href="overview.html"><strong aria-hidden="true">3.</strong> Overview</a></li><li class="chapter-item expanded "><a href="dataflow-config.html"><strong aria-hidden="true">4.</strong> Dataflow Configuration</a></li><li class="chapter-item expanded "><a href="rust-api.html"><strong aria-hidden="true">5.</strong> Rust API</a></li><li class="chapter-item expanded "><a href="c-api.html"><strong aria-hidden="true">6.</strong> C API</a></li><li class="chapter-item expanded "><a href="python-api.html"><strong aria-hidden="true">7.</strong> Python API</a></li><li class="chapter-item expanded affix "><li class="part-title">Brainstorming Ideas</li><li class="chapter-item expanded "><a href="state-management.html"><strong aria-hidden="true">8.</strong> State Management</a></li><li class="chapter-item expanded "><a href="library-vs-framework.html"><strong aria-hidden="true">9.</strong> Library vs Framework</a></li><li class="chapter-item expanded "><a href="communication-layer.html"><strong aria-hidden="true">10.</strong> Middleware Layer Abstraction</a></li></ol>            </div>
+            <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
+        </nav>
+
+        <div id="page-wrapper" class="page-wrapper">
+
+            <div class="page">
+                
+                <div id="menu-bar-hover-placeholder"></div>
+                <div id="menu-bar" class="menu-bar sticky bordered">
+                    <div class="left-buttons">
+                        <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
+                            <i class="fa fa-bars"></i>
+                        </button>
+                        <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
+                            <i class="fa fa-paint-brush"></i>
+                        </button>
+                        <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
+                            <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
+                        </ul>
+                                                <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
+                            <i class="fa fa-search"></i>
+                        </button>
+                                            </div>
+
+                    <h1 class="menu-title">dora-rs</h1>
+
+                    <div class="right-buttons">
+                                                <a href="print.html" title="Print this book" aria-label="Print this book">
+                            <i id="print-button" class="fa fa-print"></i>
+                        </a>
+                                                                        
+                    </div>
+                </div>
+
+                                <div id="search-wrapper" class="hidden">
+                    <form id="searchbar-outer" class="searchbar-outer">
+                        <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
+                    </form>
+                    <div id="searchresults-outer" class="searchresults-outer hidden">
+                        <div id="searchresults-header" class="searchresults-header"></div>
+                        <ul id="searchresults">
+                        </ul>
+                    </div>
+                </div>
+                
+                <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
+                <script type="text/javascript">
+                    document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
+                    document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
+                    Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
+                        link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
+                    });
+                </script>
+
+                <div id="content" class="content">
+                    <main>
+                        <h1 id="document-not-found-404"><a class="header" href="#document-not-found-404">Document not found (404)</a></h1>
+<p>This URL is invalid, sorry. Please use the navigation bar or search to continue.</p>
+
+                    </main>
+
+                    <nav class="nav-wrapper" aria-label="Page navigation">
+                        <!-- Mobile navigation buttons -->
+                        
+                        
+                        <div style="clear: both"></div>
+                    </nav>
+                </div>
+            </div>
+
+            <nav class="nav-wide-wrapper" aria-label="Page navigation">
+                
+                            </nav>
+
+        </div>
+
+        
+        
+        
+                <script type="text/javascript">
+            window.playground_copyable = true;
+        </script>
+        
+        
+                <script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="searcher.js" type="text/javascript" charset="utf-8"></script>
+        
+        <script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="highlight.js" type="text/javascript" charset="utf-8"></script>
+        <script src="book.js" type="text/javascript" charset="utf-8"></script>
+
+        <!-- Custom JS scripts -->
+        
+        
+    </body>
+</html>

CONTRIBUTING.md

@@ -1,59 +0,0 @@
-# How to contribute to `dora-rs`
-
-We welcome bug reports, feature requests, and pull requests!
-
-Please discuss non-trivial changes in a Github issue or on Discord first before implementing them.
-This way, we can avoid unnecessary work on both sides.
-
-## Building
-
-The `dora` project is set up as a [cargo workspace](https://doc.rust-lang.org/cargo/reference/workspaces.html).
-You can use the standard `cargo check`, `cargo build`, `cargo run`, and `cargo test` commands.
-To run a command for a specific package only, pass e.g. `--package dora-daemon`.
-Running a command for the whole workspace is possible by passing `--workspace`.
-
-
-
-## Continuous Integration (CI)
-
-We're using [GitHub Actions](https://github.com/features/actions) to run automated checks on all commits and pull requests.
-These checks ensure that our `main` branch always builds successfully and that it passes all tests.
-Please ensure that your pull request passes all checks.
-You don't need to fix warnings that are unrelated to your changes.
-Feel free to ask for help if you're unsure about a check failure.
-
-We're currently running the following kind of checks:
-
-- **CI / Test:** Ensures that the project builds and that all unit tests pass. This check is run on Linux, Windows, and macOS.
-- **CI / Examples:** Builds and runs the Rust, C, and C++ dataflows from the `examples` subdirectory. This check is run on Linux, Windows, and macOS.
-- **CI-python / Python Examples:** Builds and runs the Python dataflows from the `examples` subdirectory. This check is run on Linux only.
-- **github pages / deploy:** Generates our website from the `docs` subfolder.
-- **CI / CLI Test:** Runs some basic tests of the `dora` command-line application. This check is run on Linux, Windows, and macOS.
-- **CI / Clippy:** Runs the additional checks of the [`clippy`](https://github.com/rust-lang/rust-clippy) project.
-- **CI / Formatting:** Ensures that the code is formatted using `rustfmt` (see [below](#style))
-- **CI / License Checks:** Scans the dependency tree and tries to detect possible license incompatibilities.
-
-## Issue Management
-
-### Dora Bot
-
-We use a custom Github Action to help manage issue assignments. You can interact with this action using the following:
-
-- `@dora-bot assign me` - Assigns the current issue to you.
-- `@dora-bot unassign me` - Removes yourself from the issue assignment.
-
-For maintainers only:
-- `dora-bot unassign @username` - Allows maintainers to unassign other contributors
-Note: All issue assignments will be removed automatically after 2 weeks of inactivity.
-
-## Style
-
-We use [`rustfmt`](https://github.com/rust-lang/rustfmt) with its default settings to format our code.
-Please run `cargo fmt --all` on your code before submitting a pull request.
-Our CI will run an automatic formatting check of your code.
-
-## Publishing new Versions
-
-The maintainers are responsible for publishing new versions of the `dora` crates.
-Please don't open unsolicited pull requests to create new releases.
-Instead, request a new version by opening an issue or by leaving a comment on a merged PR.

Cargo.toml

@@ -1,203 +0,0 @@
-[workspace]
-members = [
-    "apis/c/node",
-    "apis/c/operator",
-    "apis/c++/node",
-    "apis/c++/operator",
-    "apis/python/node",
-    "apis/python/operator",
-    "apis/rust/*",
-    "apis/rust/operator/macros",
-    "apis/rust/operator/types",
-    "binaries/cli",
-    "binaries/coordinator",
-    "binaries/daemon",
-    "binaries/runtime",
-    "examples/rust-dataflow/node",
-    "examples/rust-dataflow/status-node",
-    "examples/rust-dataflow/sink",
-    "examples/rust-dataflow/sink-dynamic",
-    "examples/rust-ros2-dataflow/node",
-    "examples/benchmark/node",
-    "examples/benchmark/sink",
-    "examples/multiple-daemons/node",
-    "examples/multiple-daemons/operator",
-    "examples/multiple-daemons/sink",
-    "libraries/arrow-convert",
-    "libraries/communication-layer/*",
-    "libraries/core",
-    "libraries/message",
-    "libraries/shared-memory-server",
-    "libraries/extensions/download",
-    "libraries/extensions/telemetry/*",
-    "node-hub/dora-record",
-    "node-hub/dora-rerun",
-    "node-hub/terminal-print",
-    "node-hub/openai-proxy-server",
-    "node-hub/dora-kit-car",
-    "node-hub/dora-object-to-pose",
-    "node-hub/dora-mistral-rs",
-    "node-hub/dora-rav1e",
-    "node-hub/dora-dav1d",
-    "node-hub/dora-rustypot",
-    "libraries/extensions/ros2-bridge",
-    "libraries/extensions/ros2-bridge/msg-gen",
-    "libraries/extensions/ros2-bridge/python",
-    "tests/queue_size_latest_data_rust/receive_data",
-
-]
-
-[workspace.package]
-edition = "2024"
-rust-version = "1.85.0"
-# Make sure to also bump `apis/node/python/__init__.py` version.
-version = "0.3.12"
-description = "`dora` goal is to be a low latency, composable, and distributed data flow."
-documentation = "https://dora-rs.ai"
-license = "Apache-2.0"
-repository = "https://github.com/dora-rs/dora/"
-
-[workspace.dependencies]
-dora-node-api = { version = "0.3.12", path = "apis/rust/node", default-features = false }
-dora-node-api-python = { version = "0.3.12", path = "apis/python/node", default-features = false }
-dora-operator-api = { version = "0.3.12", path = "apis/rust/operator", default-features = false }
-dora-operator-api-macros = { version = "0.3.12", path = "apis/rust/operator/macros" }
-dora-operator-api-types = { version = "0.3.12", path = "apis/rust/operator/types" }
-dora-operator-api-python = { version = "0.3.12", path = "apis/python/operator" }
-dora-operator-api-c = { version = "0.3.12", path = "apis/c/operator" }
-dora-node-api-c = { version = "0.3.12", path = "apis/c/node" }
-dora-core = { version = "0.3.12", path = "libraries/core" }
-dora-arrow-convert = { version = "0.3.12", path = "libraries/arrow-convert" }
-dora-tracing = { version = "0.3.12", path = "libraries/extensions/telemetry/tracing" }
-dora-metrics = { version = "0.3.12", path = "libraries/extensions/telemetry/metrics" }
-dora-download = { version = "0.3.12", path = "libraries/extensions/download" }
-shared-memory-server = { version = "0.3.12", path = "libraries/shared-memory-server" }
-communication-layer-request-reply = { version = "0.3.12", path = "libraries/communication-layer/request-reply" }
-dora-cli = { version = "0.3.12", path = "binaries/cli" }
-dora-runtime = { version = "0.3.12", path = "binaries/runtime" }
-dora-daemon = { version = "0.3.12", path = "binaries/daemon" }
-dora-coordinator = { version = "0.3.12", path = "binaries/coordinator" }
-dora-ros2-bridge = { version = "0.3.12", path = "libraries/extensions/ros2-bridge" }
-dora-ros2-bridge-msg-gen = { version = "0.3.12", path = "libraries/extensions/ros2-bridge/msg-gen" }
-dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" }
-# versioned independently from the other dora crates
-dora-message = { version = "0.5.0", path = "libraries/message" }
-arrow = { version = "54.2.1" }
-arrow-schema = { version = "54.2.1" }
-arrow-data = { version = "54.2.1" }
-arrow-array = { version = "54.2.1" }
-parquet = { version = "54.2.1" }
-pyo3 = { version = "0.23", features = [
-    "eyre",
-    "abi3-py37",
-    "multiple-pymethods",
-] }
-pythonize = "0.23"
-git2 = { version = "0.18.0", features = ["vendored-openssl"] }
-serde_yaml = "0.9.33"
-
-[package]
-name = "dora-examples"
-rust-version = "1.85.0"
-version = "0.0.0"
-edition.workspace = true
-license = "Apache-2.0"
-publish = false
-
-
-[features]
-# enables examples that depend on a sourced ROS2 installation
-ros2-examples = []
-
-[dev-dependencies]
-eyre = "0.6.8"
-tokio = "1.24.2"
-dora-cli = { workspace = true }
-dora-coordinator = { workspace = true }
-dora-core = { workspace = true }
-dora-message = { workspace = true }
-dora-tracing = { workspace = true }
-dora-download = { workspace = true }
-dunce = "1.0.2"
-serde_yaml = "0.8.23"
-uuid = { version = "1.7", features = ["v7", "serde"] }
-tracing = "0.1.36"
-futures = "0.3.25"
-tokio-stream = "0.1.11"
-
-[[example]]
-name = "c-dataflow"
-path = "examples/c-dataflow/run.rs"
-
-[[example]]
-name = "camera"
-path = "examples/camera/run.rs"
-
-[[example]]
-name = "vlm"
-path = "examples/vlm/run.rs"
-
-[[example]]
-name = "rust-dataflow"
-path = "examples/rust-dataflow/run.rs"
-
-[[example]]
-name = "rust-dataflow-git"
-path = "examples/rust-dataflow-git/run.rs"
-
-[[example]]
-name = "rust-ros2-dataflow"
-path = "examples/rust-ros2-dataflow/run.rs"
-required-features = ["ros2-examples"]
-
-# TODO: Fix example #192
-[[example]]
-name = "rust-dataflow-url"
-path = "examples/rust-dataflow-url/run.rs"
-
-[[example]]
-name = "cxx-dataflow"
-path = "examples/c++-dataflow/run.rs"
-
-[[example]]
-name = "cxx-arrow-dataflow"
-path = "examples/c++-arrow-dataflow/run.rs"
-
-[[example]]
-name = "python-dataflow"
-path = "examples/python-dataflow/run.rs"
-
-[[example]]
-name = "python-ros2-dataflow"
-path = "examples/python-ros2-dataflow/run.rs"
-required-features = ["ros2-examples"]
-
-[[example]]
-name = "python-operator-dataflow"
-path = "examples/python-operator-dataflow/run.rs"
-
-[[example]]
-name = "benchmark"
-path = "examples/benchmark/run.rs"
-
-[[example]]
-name = "multiple-daemons"
-path = "examples/multiple-daemons/run.rs"
-
-[[example]]
-name = "cmake-dataflow"
-path = "examples/cmake-dataflow/run.rs"
-
-[[example]]
-name = "cxx-ros2-dataflow"
-path = "examples/c++-ros2-dataflow/run.rs"
-required-features = ["ros2-examples"]
-
-[[example]]
-name = "rerun-viewer"
-path = "examples/rerun-viewer/run.rs"
-
-# The profile that 'dist' will build with
-[profile.dist]
-inherits = "release"
-lto = "thin"

Changelog.md

@@ -1,682 +0,0 @@
-# Changelog
-
-## v0.3.12 (2025-06-30)
-
-## What's Changed
-
-- Implemented dora-cotracker node by @ShashwatPatil in https://github.com/dora-rs/dora/pull/931
-- Minor fix and add boxes2d example to facebook/cotracker by @haixuanTao in https://github.com/dora-rs/dora/pull/950
-- Update Rust crate tokio to v1.44.2 [SECURITY] by @renovate in https://github.com/dora-rs/dora/pull/951
-- Post 3.11 release fix by @haixuanTao in https://github.com/dora-rs/dora/pull/954
-- Bump crossbeam-channel from 0.5.14 to 0.5.15 by @dependabot in https://github.com/dora-rs/dora/pull/959
-- Added E ruff flag for pydocstyle by @7SOMAY in https://github.com/dora-rs/dora/pull/958
-- Revert "Added E ruff flag for better code quality [skip ci]" by @haixuanTao in https://github.com/dora-rs/dora/pull/968
-- Ease of use changes in benches for issue #957 by @Ignavar in https://github.com/dora-rs/dora/pull/969
-- Reachy cotracker by @haixuanTao in https://github.com/dora-rs/dora/pull/972
-- Improve rav1e by @haixuanTao in https://github.com/dora-rs/dora/pull/974
-- Fix pyrealsense by @haixuanTao in https://github.com/dora-rs/dora/pull/973
-- Added Self Uninstall Command by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/944
-- Improve benchmark implementation & Add warning for discarding events by @Mivik in https://github.com/dora-rs/dora/pull/971
-- docs: Updated README: Added comprehensive usage documentation with vi… by @LeonRust in https://github.com/dora-rs/dora/pull/983
-- Fix rerun-viewer example. by @francocipollone in https://github.com/dora-rs/dora/pull/989
-- docs: add license badge by @Radovenchyk in https://github.com/dora-rs/dora/pull/996
-- Disable sccache for `musllinux` jobs by @haixuanTao in https://github.com/dora-rs/dora/pull/1000
-- Remove unused sysinfo monitor by @Mivik in https://github.com/dora-rs/dora/pull/1007
-- Refactor Python CUDA IPC API by @Mivik in https://github.com/dora-rs/dora/pull/1002
-- fix terminal not printing stdout on nvml warning by @haixuanTao in https://github.com/dora-rs/dora/pull/1008
-- Fix issue #1006: [Brief description of the fix] by @sohamukute in https://github.com/dora-rs/dora/pull/1013
-- Improving so100 usability by @haixuanTao in https://github.com/dora-rs/dora/pull/988
-- Add dora-mediapipe node for quick human pose estimation by @haixuanTao in https://github.com/dora-rs/dora/pull/986
-- Bump torch to 2.7 by @haixuanTao in https://github.com/dora-rs/dora/pull/1015
-- refactor(tracing): use builder style by @sjfhsjfh in https://github.com/dora-rs/dora/pull/1009
-- Fix spawning runtime through python when it is installed with pip by @haixuanTao in https://github.com/dora-rs/dora/pull/1011
-- chore(deps): update dependency numpy to v2 by @renovate in https://github.com/dora-rs/dora/pull/1014
-- Fix error when multiple visualization key is active and when urdf_transform env variable is not present by @haixuanTao in https://github.com/dora-rs/dora/pull/1016
-- Update pyrealsense2 Dependencies for L515 Support and Fix README wget Link by @kingchou007 in https://github.com/dora-rs/dora/pull/1021
-- Minor fix for mujoco sim by @haixuanTao in https://github.com/dora-rs/dora/pull/1023
-- dora-mujoco simulation node with example for controlling any arm by @ShashwatPatil in https://github.com/dora-rs/dora/pull/1012
-- fix ros CI/CD by @haixuanTao in https://github.com/dora-rs/dora/pull/1027
-- dora-vggt by @haixuanTao in https://github.com/dora-rs/dora/pull/1024
-- Adding vision to openai server by @haixuanTao in https://github.com/dora-rs/dora/pull/1025
-- Revert "Adding vision to openai server" by @haixuanTao in https://github.com/dora-rs/dora/pull/1031
-- Expose AllInputClosed message as a Stop message by @haixuanTao in https://github.com/dora-rs/dora/pull/1026
-- Add support for git repository sources for nodes by @phil-opp in https://github.com/dora-rs/dora/pull/901
-- Adding vision to rust openai proxy server by @haixuanTao in https://github.com/dora-rs/dora/pull/1033
-- Add automatic robot descriptions URDF retrieval from https://github.com/robot-descriptions/robot_descriptions.py by @haixuanTao in https://github.com/dora-rs/dora/pull/1032
-
-## New Contributors
-
-- @Mivik made their first contribution in https://github.com/dora-rs/dora/pull/971
-- @francocipollone made their first contribution in https://github.com/dora-rs/dora/pull/989
-- @sohamukute made their first contribution in https://github.com/dora-rs/dora/pull/1013
-- @sjfhsjfh made their first contribution in https://github.com/dora-rs/dora/pull/1009
-- @kingchou007 made their first contribution in https://github.com/dora-rs/dora/pull/1021
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.11...v0.3.12
-
-## v0.3.11 (2025-04-07)
-
-## What's Changed
-
-- Post dora 0.3.10 release fix by @haixuanTao in https://github.com/dora-rs/dora/pull/804
-- Add windows release for rust nodes by @haixuanTao in https://github.com/dora-rs/dora/pull/805
-- Add Node Table into README.md by @haixuanTao in https://github.com/dora-rs/dora/pull/808
-- update dora yaml json schema validator by @haixuanTao in https://github.com/dora-rs/dora/pull/809
-- Improve readme support matrix readability by @haixuanTao in https://github.com/dora-rs/dora/pull/810
-- Clippy automatic fixes applied by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/812
-- Improve documentation on adding new node to the node-hub by @haixuanTao in https://github.com/dora-rs/dora/pull/820
-- #807 Fixed by @7SOMAY in https://github.com/dora-rs/dora/pull/818
-- Applied Ruff pydocstyle to dora by @Mati-ur-rehman-017 in https://github.com/dora-rs/dora/pull/831
-- Related to dora-bot issue assignment by @MunishMummadi in https://github.com/dora-rs/dora/pull/840
-- Add dora-lerobot node into dora by @Ignavar in https://github.com/dora-rs/dora/pull/834
-- CI: Permit issue modifications for issue assign job by @phil-opp in https://github.com/dora-rs/dora/pull/848
-- Fix: Set variables outside bash script to prevent injection by @phil-opp in https://github.com/dora-rs/dora/pull/849
-- Replacing Deprecated functions of pyo3 by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/838
-- Add noise filtering on whisper to be able to use speakers by @haixuanTao in https://github.com/dora-rs/dora/pull/847
-- Add minimal Dockerfile with Python and uv for easy onboarding by @Krishnadubey1008 in https://github.com/dora-rs/dora/pull/843
-- More compact readme with example section by @haixuanTao in https://github.com/dora-rs/dora/pull/855
-- Create docker-image.yml by @haixuanTao in https://github.com/dora-rs/dora/pull/857
-- Multi platform docker by @haixuanTao in https://github.com/dora-rs/dora/pull/858
-- change: `dora/node-hub/README.md` by @MunishMummadi in https://github.com/dora-rs/dora/pull/862
-- Added dora-phi4 inside node-hub by @7SOMAY in https://github.com/dora-rs/dora/pull/861
-- node-hub: Added dora-magma node by @MunishMummadi in https://github.com/dora-rs/dora/pull/853
-- Added the dora-llama-cpp-python node by @ShashwatPatil in https://github.com/dora-rs/dora/pull/850
-- Adding in some missing types and test cases within arrow convert crate by @Ignavar in https://github.com/dora-rs/dora/pull/864
-- Migrate robots from dora-lerobot to dora repository by @rahat2134 in https://github.com/dora-rs/dora/pull/868
-- Applied pyupgrade style by @Mati-ur-rehman-017 in https://github.com/dora-rs/dora/pull/876
-- Adding additional llm in tests by @haixuanTao in https://github.com/dora-rs/dora/pull/873
-- Dora transformer node by @ShashwatPatil in https://github.com/dora-rs/dora/pull/870
-- Using macros in Arrow Conversion by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/877
-- Adding run command within python API by @haixuanTao in https://github.com/dora-rs/dora/pull/875
-- Added f16 type conversion by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/886
-- Added "PERF" flag inside node-hub by @7SOMAY in https://github.com/dora-rs/dora/pull/880
-- Added quality ruff-flags for better code quality by @7SOMAY in https://github.com/dora-rs/dora/pull/888
-- Add llm benchmark by @haixuanTao in https://github.com/dora-rs/dora/pull/881
-- Implement `into_vec_f64(&ArrowData) -> Vec<f64)` conversion function by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/893
-- Adding virtual env within dora build command by @haixuanTao in https://github.com/dora-rs/dora/pull/895
-- Adding metrics for node api by @haixuanTao in https://github.com/dora-rs/dora/pull/903
-- Made UI interface for input in dora, using Gradio by @ShashwatPatil in https://github.com/dora-rs/dora/pull/891
-- Add chinese voice support by @haixuanTao in https://github.com/dora-rs/dora/pull/902
-- Made conversion generic by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/908
-- Added husky simulation in Mujoco and gamepad node by @ShashwatPatil in https://github.com/dora-rs/dora/pull/906
-- use `cargo-dist` tool for dora-cli releases by @Hennzau in https://github.com/dora-rs/dora/pull/916
-- Implementing Self update by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/920
-- Fix: RUST_LOG=. dora run bug by @starlitxiling in https://github.com/dora-rs/dora/pull/924
-- Added dora-mistral-rs node in node-hub for inference in rust by @Ignavar in https://github.com/dora-rs/dora/pull/910
-- Fix reachy left arm by @haixuanTao in https://github.com/dora-rs/dora/pull/907
-- Functions for sending and receiving data using Arrow::FFI by @Mati-ur-rehman-017 in https://github.com/dora-rs/dora/pull/918
-- Adding `recv_async` dora method to retrieve data in python async by @haixuanTao in https://github.com/dora-rs/dora/pull/909
-- Update: README.md of the node hub by @Choudhry18 in https://github.com/dora-rs/dora/pull/929
-- Fix magma by @haixuanTao in https://github.com/dora-rs/dora/pull/926
-- Add support for mask in rerun by @haixuanTao in https://github.com/dora-rs/dora/pull/927
-- Bump array-init-cursor from 0.2.0 to 0.2.1 by @dependabot in https://github.com/dora-rs/dora/pull/933
-- Enhance Zenoh Integration Documentation by @NageshMandal in https://github.com/dora-rs/dora/pull/935
-- Support av1 by @haixuanTao in https://github.com/dora-rs/dora/pull/932
-- Bump dora v0.3.11 by @haixuanTao in https://github.com/dora-rs/dora/pull/948
-
-## New Contributors
-
-- @Shar-jeel-Sajid made their first contribution in https://github.com/dora-rs/dora/pull/812
-- @7SOMAY made their first contribution in https://github.com/dora-rs/dora/pull/818
-- @Mati-ur-rehman-017 made their first contribution in https://github.com/dora-rs/dora/pull/831
-- @MunishMummadi made their first contribution in https://github.com/dora-rs/dora/pull/840
-- @Ignavar made their first contribution in https://github.com/dora-rs/dora/pull/834
-- @Krishnadubey1008 made their first contribution in https://github.com/dora-rs/dora/pull/843
-- @ShashwatPatil made their first contribution in https://github.com/dora-rs/dora/pull/850
-- @rahat2134 made their first contribution in https://github.com/dora-rs/dora/pull/868
-- @Choudhry18 made their first contribution in https://github.com/dora-rs/dora/pull/929
-- @NageshMandal made their first contribution in https://github.com/dora-rs/dora/pull/935
-
-## v0.3.10 (2025-03-04)
-
-## What's Changed
-
-- Enables array based bounding boxes by @haixuanTao in https://github.com/dora-rs/dora/pull/772
-- Fix typo in node version by @haixuanTao in https://github.com/dora-rs/dora/pull/773
-- CI: Use `paths-ignore` instead of negated `paths` by @phil-opp in https://github.com/dora-rs/dora/pull/781
-- Adding rerun connect options by @haixuanTao in https://github.com/dora-rs/dora/pull/782
-- Forbid `/` in node IDs by @phil-opp in https://github.com/dora-rs/dora/pull/785
-- Adding reachy and dora reachy demo by @haixuanTao in https://github.com/dora-rs/dora/pull/784
-- Fix typo in reachy node by @haixuanTao in https://github.com/dora-rs/dora/pull/789
-- Update dependency transformers to >=4.48.0,<=4.48.0 [SECURITY] - abandoned by @renovate in https://github.com/dora-rs/dora/pull/778
-- Fix bounding box for rerun viewer and clear the viewer if no bounding box is detected by @haixuanTao in https://github.com/dora-rs/dora/pull/787
-- Adding float for env variable and metadata parameters by @haixuanTao in https://github.com/dora-rs/dora/pull/786
-- Limit pip release ci to strict minimum by @haixuanTao in https://github.com/dora-rs/dora/pull/791
-- Add uv flag within start cli command by @haixuanTao in https://github.com/dora-rs/dora/pull/788
-- Adding a test for checking on the latency when used timeout and queue at the same time by @haixuanTao in https://github.com/dora-rs/dora/pull/783
-- Use zenoh for inter-daemon communication by @phil-opp in https://github.com/dora-rs/dora/pull/779
-- Pin chrono version by @haixuanTao in https://github.com/dora-rs/dora/pull/797
-- Add kokoro tts by @haixuanTao in https://github.com/dora-rs/dora/pull/794
-- Pick place demo by @haixuanTao in https://github.com/dora-rs/dora/pull/793
-- Bump pyo3 to 0.23 by @haixuanTao in https://github.com/dora-rs/dora/pull/798
-- Faster node hub CI/CD by removing `free disk space on ubuntu` by @haixuanTao in https://github.com/dora-rs/dora/pull/801
-
-## v0.3.9 (2025-02-06)
-
-## What's Changed
-
-- Making cli install the default api avoiding confusion on install by @haixuanTao in https://github.com/dora-rs/dora/pull/739
-- Add description within visualisation by @haixuanTao in https://github.com/dora-rs/dora/pull/742
-- Added depth image and data output for the dora-pyorbbecksdk node by @Ryu-Yang in https://github.com/dora-rs/dora/pull/740
-- Improve speech to text example within the macOS ecosystem by @haixuanTao in https://github.com/dora-rs/dora/pull/741
-- Rewrite python template to make them pip installable by @haixuanTao in https://github.com/dora-rs/dora/pull/744
-- bump rerun version by @haixuanTao in https://github.com/dora-rs/dora/pull/743
-- Replace pylint with ruff by @haixuanTao in https://github.com/dora-rs/dora/pull/756
-- Make unknown output acceptable by @haixuanTao in https://github.com/dora-rs/dora/pull/755
-- Improve Speech-to-Speech pipeline by better support for macOS and additional OutteTTS model by @haixuanTao in https://github.com/dora-rs/dora/pull/752
-- Daemon: React to ctrl-c during connection setup by @phil-opp in https://github.com/dora-rs/dora/pull/758
-- Use UV for the CI/CD by @haixuanTao in https://github.com/dora-rs/dora/pull/757
-- chore: fix some typos in comment by @sunxunle in https://github.com/dora-rs/dora/pull/759
-- Add ios lidar by @haixuanTao in https://github.com/dora-rs/dora/pull/762
-- Print python stdout without buffer even for script by @haixuanTao in https://github.com/dora-rs/dora/pull/761
-- chore: use workspace edition by @yjhmelody in https://github.com/dora-rs/dora/pull/764
-- Add a uv flag to make it possible to automatically replace `pip` with `uv pip` and prepend run command with `uv run` by @haixuanTao in https://github.com/dora-rs/dora/pull/765
-- Use mlx whisper instead of lightning-whisper by @haixuanTao in https://github.com/dora-rs/dora/pull/766
-- Reduce silence duration in VAD by @haixuanTao in https://github.com/dora-rs/dora/pull/768
-- Bump upload artifact version by @haixuanTao in https://github.com/dora-rs/dora/pull/769
-- Add qwenvl2 5 by @haixuanTao in https://github.com/dora-rs/dora/pull/767
-
-## New Contributors
-
-- @Ryu-Yang made their first contribution in https://github.com/dora-rs/dora/pull/740
-- @sunxunle made their first contribution in https://github.com/dora-rs/dora/pull/759
-- @yjhmelody made their first contribution in https://github.com/dora-rs/dora/pull/764
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.8...v0.3.9
-
-## Breaking Change
-
-Inputs are now schedule fairly meaning that they will be now be received equally and not necessarily in chronological order. This enables to always be able to refresh input with the least latency between input.
-
-## v0.3.8 (2024-12-06)
-
-- Make node hub CI/CD cross platform by @haixuanTao in https://github.com/dora-rs/dora/pull/714
-- Make node hub CI/CD cross architecture by @haixuanTao in https://github.com/dora-rs/dora/pull/716
-- Make list an available type for metadata by @haixuanTao in https://github.com/dora-rs/dora/pull/721
-- Add stdout logging by @haixuanTao in https://github.com/dora-rs/dora/pull/720
-- Add an error when a node fails when using dora run by @haixuanTao in https://github.com/dora-rs/dora/pull/719
-- Add pyarrow cuda zero copy helper by @haixuanTao in https://github.com/dora-rs/dora/pull/722
-- feat: Add Dora-kit car Control in node-hub by @LyonRust in https://github.com/dora-rs/dora/pull/715
-- Add yuv420 encoding to opencv-video-capture by @haixuanTao in https://github.com/dora-rs/dora/pull/725
-- Change macOS CI runner to `macos-13` by @phil-opp in https://github.com/dora-rs/dora/pull/729
-- Add eyre to pyo3 node by @haixuanTao in https://github.com/dora-rs/dora/pull/730
-- Moving queue size and making node flume queue bigger by @haixuanTao in https://github.com/dora-rs/dora/pull/724
-- Make python default for macos by @haixuanTao in https://github.com/dora-rs/dora/pull/731
-- Modify the node queue Scheduler to make it able to schedule input fairly by @haixuanTao in https://github.com/dora-rs/dora/pull/728
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.7...v0.3.8
-
-## v0.3.7 (2024-11-15)
-
-## What's Changed
-
-- Post release `0.3.6` small fix by @haixuanTao in https://github.com/dora-rs/dora/pull/638
-- Changes to template by @XxChang in https://github.com/dora-rs/dora/pull/639
-- Add appending to PATH instruction inside installation script by @Hennzau in https://github.com/dora-rs/dora/pull/641
-- Make the benchmark run in release and at full speed by @Hennzau in https://github.com/dora-rs/dora/pull/644
-- Use the new node syntax for examples dataflow by @Hennzau in https://github.com/dora-rs/dora/pull/643
-- Improve beginner experience by @Hennzau in https://github.com/dora-rs/dora/pull/645
-- improve node-hub pytest by @haixuanTao in https://github.com/dora-rs/dora/pull/640
-- Fix not-null terminated string print within C template by @haixuanTao in https://github.com/dora-rs/dora/pull/654
-- Raise error if dora-coordinator is not connected when calling `dora destroy` by @haixuanTao in https://github.com/dora-rs/dora/pull/655
-- Coordinator stopped on bad control command by @Hennzau in https://github.com/dora-rs/dora/pull/650
-- Add support for Qwenvl2 by @haixuanTao in https://github.com/dora-rs/dora/pull/646
-- Fix distributed node by @haixuanTao in https://github.com/dora-rs/dora/pull/658
-- Small install script update for bash by @haixuanTao in https://github.com/dora-rs/dora/pull/657
-- Add additional image encoding by @haixuanTao in https://github.com/dora-rs/dora/pull/661
-- `dora-echo` replicate the topic received with the topic send by @haixuanTao in https://github.com/dora-rs/dora/pull/663
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/656
-- Bump pyo3 and arrow versions by @haixuanTao in https://github.com/dora-rs/dora/pull/667
-- Fix ros2 bridge incompatibility with CI Ubuntu 24 and with pyo3 22 by @haixuanTao in https://github.com/dora-rs/dora/pull/670
-- Add transformers version pinning for qwenvl2 by @haixuanTao in https://github.com/dora-rs/dora/pull/665
-- Remove cli dataflow path check by @haixuanTao in https://github.com/dora-rs/dora/pull/662
-- Better error handling for unknown output by @haixuanTao in https://github.com/dora-rs/dora/pull/675
-- Fix llama recorder multi image recorder by @haixuanTao in https://github.com/dora-rs/dora/pull/677
-- Dora openai server example by @haixuanTao in https://github.com/dora-rs/dora/pull/676
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/674
-- Create Rust-based openai api proxy server in node hub by @phil-opp in https://github.com/dora-rs/dora/pull/678
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/679
-- Update Rust crate hyper to v0.14.30 by @renovate in https://github.com/dora-rs/dora/pull/680
-- Fix hanged coordinator when failing to connect to the daemon on destroy command by @haixuanTao in https://github.com/dora-rs/dora/pull/664
-- Small example improvement using pyarrow assertion by @haixuanTao in https://github.com/dora-rs/dora/pull/669
-- Fix dora list listing twice a stopping dataflow when using multiple daemon. by @haixuanTao in https://github.com/dora-rs/dora/pull/668
-- Add package flake by @Ben-PH in https://github.com/dora-rs/dora/pull/685
-- Add jpeg format to qwenvl2 by @haixuanTao in https://github.com/dora-rs/dora/pull/684
-- Enable downloading remote dataflow by @haixuanTao in https://github.com/dora-rs/dora/pull/682
-- Enable multiline build for better packaging of dora node. by @haixuanTao in https://github.com/dora-rs/dora/pull/683
-- Bump rerun version to 0.18 by @haixuanTao in https://github.com/dora-rs/dora/pull/686
-- Temporary fix qwenvl2 queue error by @haixuanTao in https://github.com/dora-rs/dora/pull/688
-- Make daemon loop over coordinator connection to make it possible to create a system service awaiting coordinator connection by @haixuanTao in https://github.com/dora-rs/dora/pull/689
-- Add translation example from chinese, french to english by @haixuanTao in https://github.com/dora-rs/dora/pull/681
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/690
-- Fix macos 14 yolo error by @haixuanTao in https://github.com/dora-rs/dora/pull/696
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/692
-- Publish rust project on pip to make it simpler to deploy dora node on different machine without requiring installing cargo by @haixuanTao in https://github.com/dora-rs/dora/pull/695
-- Docs: README by @Radovenchyk in https://github.com/dora-rs/dora/pull/697
-- Update README.md by @pucedoteth in https://github.com/dora-rs/dora/pull/705
-- Bump rust toolchains 1.81 by @haixuanTao in https://github.com/dora-rs/dora/pull/707
-- Make dora cli pip installable by @haixuanTao in https://github.com/dora-rs/dora/pull/706
-- Add urdf visualization in rerun by @haixuanTao in https://github.com/dora-rs/dora/pull/704
-- Fix child process receiving ctrl-c by setting own process group by @haixuanTao in https://github.com/dora-rs/dora/pull/712
-- Move more types from `dora-core` to `dora-message` to avoid dependency by @phil-opp in https://github.com/dora-rs/dora/pull/711
-- Implement `dora run` command by @phil-opp in https://github.com/dora-rs/dora/pull/703
-- Adding Agilex Piper node, PyOrbbeckSDK node, Agilex UGV node by @haixuanTao in https://github.com/dora-rs/dora/pull/709
-- Make the node hub CI/CD parallel for faster testing as well as having more granular integration control by @haixuanTao in https://github.com/dora-rs/dora/pull/710
-- Add time series to dora rerun by @haixuanTao in https://github.com/dora-rs/dora/pull/713
-
-## New Contributors
-
-- @Ben-PH made their first contribution in https://github.com/dora-rs/dora/pull/685
-- @Radovenchyk made their first contribution in https://github.com/dora-rs/dora/pull/697
-- @pucedoteth made their first contribution in https://github.com/dora-rs/dora/pull/705
-
-## v0.3.6 (2024-08-17)
-
-## What's Changed
-
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/579
-- Don't wait for non-started dynamic nodes on stop by @phil-opp in https://github.com/dora-rs/dora/pull/583
-- add a comment on read_dora_input_id by @XxChang in https://github.com/dora-rs/dora/pull/580
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/584
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/585
-- Add domain unix socket supports by @XxChang in https://github.com/dora-rs/dora/pull/594
-- Check build for cross-compiled targets on CI by @phil-opp in https://github.com/dora-rs/dora/pull/597
-- Test pip release creation as part of normal CI by @phil-opp in https://github.com/dora-rs/dora/pull/596
-- Add-armv7-musleabihf-prebuilt-release by @haixuanTao in https://github.com/dora-rs/dora/pull/578
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/602
-- Delay dropping of `DoraNode` in Python until all event data is freed by @phil-opp in https://github.com/dora-rs/dora/pull/601
-- Add install script by @haixuanTao in https://github.com/dora-rs/dora/pull/600
-- Nodes hub to store and reuse commonly used node by @haixuanTao in https://github.com/dora-rs/dora/pull/569
-- Ros2-bridge action attempt by @starlitxiling in https://github.com/dora-rs/dora/pull/567
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/605
-- Add a CI/CD for the node-hub by @haixuanTao in https://github.com/dora-rs/dora/pull/604
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/608
-- Remove dynamic node from pending nodes before starting a dataflow by @haixuanTao in https://github.com/dora-rs/dora/pull/606
-- Fix alignment of atomics in shared memory communication channel by @phil-opp in https://github.com/dora-rs/dora/pull/612
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/622
-- Refactor: Move message definitions to `dora-message` crate by @phil-opp in https://github.com/dora-rs/dora/pull/613
-- Update README.md by @heyong4725 in https://github.com/dora-rs/dora/pull/623
-- Update Rust crate serde to v1.0.207 by @renovate in https://github.com/dora-rs/dora/pull/624
-- fix clippy warnings by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/626
-
-## v0.3.5 (2024-07-03)
-
-## What's Changed
-
-- chore: Support RISCV64 by @LyonRust in https://github.com/dora-rs/dora/pull/505
-- Json schemas for VSCode YAML Support by @haixuanTao in https://github.com/dora-rs/dora/pull/497
-- Pretty Print Rust object when called from Python print by @haixuanTao in https://github.com/dora-rs/dora/pull/503
-- Fix `Cargo.lock` by @phil-opp in https://github.com/dora-rs/dora/pull/506
-- Use dependabot for automatic lockfile updates by @phil-opp in https://github.com/dora-rs/dora/pull/507
-- Run cargo update by @phil-opp in https://github.com/dora-rs/dora/pull/508
-- Allow top-level fields in node declaration by @phil-opp in https://github.com/dora-rs/dora/pull/478
-- Configure Renovate by @renovate in https://github.com/dora-rs/dora/pull/509
-- Make non-UTF8 stdout/stderr from nodes non-fatal by @phil-opp in https://github.com/dora-rs/dora/pull/510
-- Make dora cli connect to remote coordinator by @Gege-Wang in https://github.com/dora-rs/dora/pull/513
-- Provide help messages for CLI by @phil-opp in https://github.com/dora-rs/dora/pull/519
-- Renovate: group all dependency updates in single PR by @phil-opp in https://github.com/dora-rs/dora/pull/524
-- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/529
-- Improve coordinator port config by @phil-opp in https://github.com/dora-rs/dora/pull/520
-- Fix some typos and add automatic typos check to CI by @EricLBuehler in https://github.com/dora-rs/dora/pull/539
-- Update Pyo3 bounds by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/472
-- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/543
-- Small logging improvements by @phil-opp in https://github.com/dora-rs/dora/pull/537
-- Refuse relative path for remote in coordinator by @XxChang in https://github.com/dora-rs/dora/pull/538
-- chore(deps): update rust crate clap to v4.5.7 by @renovate in https://github.com/dora-rs/dora/pull/546
-- Add `--quiet` flag to daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/548
-- Implement file-based logging in daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/549
-- Spawn daemon and coordinator in quiet mode on `dora up` by @phil-opp in https://github.com/dora-rs/dora/pull/550
-- Run dynamic node by @haixuanTao in https://github.com/dora-rs/dora/pull/517
-- Update dora new by @XxChang in https://github.com/dora-rs/dora/pull/553
-- fix event_as_input bug by @XxChang in https://github.com/dora-rs/dora/pull/556
-- Transform custom PyEvent into standard python dictionary for easier d… by @haixuanTao in https://github.com/dora-rs/dora/pull/557
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/558
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/560
-- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/563
-- Print only first node error and report more metadata in dataflow results by @phil-opp in https://github.com/dora-rs/dora/pull/552
-- Make `dora start` attach by default, add `--detach` to opt-out by @phil-opp in https://github.com/dora-rs/dora/pull/561
-- List failed and finished dataflows in `dora list` by @phil-opp in https://github.com/dora-rs/dora/pull/554
-- Ignore-quicker-pending-drop-token by @haixuanTao in https://github.com/dora-rs/dora/pull/568
-- Increasing grace duration to 2 seconds so that drop token get well returned in https://github.com/dora-rs/dora/pull/576
-
-## New Contributors
-
-- @LyonRust made their first contribution in https://github.com/dora-rs/dora/pull/505
-- @renovate made their first contribution in https://github.com/dora-rs/dora/pull/509
-- @Gege-Wang made their first contribution in https://github.com/dora-rs/dora/pull/513
-- @EricLBuehler made their first contribution in https://github.com/dora-rs/dora/pull/539
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.4...v0.3.5
-
-## v0.3.4 (2024-05-17)
-
-## What's Changed
-
-- Remove `cxx_build` call, which is no longer used by @phil-opp in https://github.com/dora-rs/dora/pull/470
-- Update `ros2-client` to latest version by @phil-opp in https://github.com/dora-rs/dora/pull/457
-- Configurable bind addrs by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/471
-- Simple warning fixes by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/477
-- Adding `dora-rerun` as a visualization tool by @haixuanTao in https://github.com/dora-rs/dora/pull/479
-- Fix Clippy and RERUN_MEMORY_LIMIT env variable default by @haixuanTao in https://github.com/dora-rs/dora/pull/490
-- Fix CI build errors by @phil-opp in https://github.com/dora-rs/dora/pull/491
-- Use `resolver = 2` for in workspace in Rust template by @phil-opp in https://github.com/dora-rs/dora/pull/492
-- Add grace duration and kill process by @haixuanTao in https://github.com/dora-rs/dora/pull/487
-- Simplify parsing of `AMENT_PREFIX_PATH` by @haixuanTao in https://github.com/dora-rs/dora/pull/489
-- Convert rust example to node by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/494
-- Adding python IDE typing by @haixuanTao in https://github.com/dora-rs/dora/pull/493
-- Fix: Wait until dora daemon is connected to coordinator on `dora up` by @phil-opp in https://github.com/dora-rs/dora/pull/496
-
-## New Contributors
-
-- @Michael-J-Ward made their first contribution in https://github.com/dora-rs/dora/pull/471
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.3...v0.3.4
-
-## v0.3.3 (2024-04-08)
-
-## What's Changed
-
-- Metrics refactoring by @haixuanTao in https://github.com/dora-rs/dora/pull/423
-- Add ROS2 bridge support for C++ nodes by @phil-opp in https://github.com/dora-rs/dora/pull/425
-- Provide function to create empty `CombinedEvents` stream by @phil-opp in https://github.com/dora-rs/dora/pull/432
-- Expose ROS2 constants in generated bindings (Rust and C++) by @phil-opp in https://github.com/dora-rs/dora/pull/428
-- Add option to send `stdout` as node/operator output by @haixuanTao in https://github.com/dora-rs/dora/pull/388
-- Fix warning about `#pragma once` in main file by @phil-opp in https://github.com/dora-rs/dora/pull/433
-- Send runs artefacts into a dedicated `out` folder by @haixuanTao in https://github.com/dora-rs/dora/pull/429
-- Create README.md for cxx-ros2-example by @bobd988 in https://github.com/dora-rs/dora/pull/431
-- Use Async Parquet Writer for `dora-record` by @haixuanTao in https://github.com/dora-rs/dora/pull/434
-- Update mio to fix security vulnerability by @phil-opp in https://github.com/dora-rs/dora/pull/440
-- Add initial support for calling ROS2 services from Rust nodes by @phil-opp in https://github.com/dora-rs/dora/pull/439
-- Enable ROS2 service calls from C++ nodes by @phil-opp in https://github.com/dora-rs/dora/pull/441
-- Use `Debug` formatting for eyre errors when returning to C++ by @phil-opp in https://github.com/dora-rs/dora/pull/450
-- Fix out-of-tree builds in cmake example by @phil-opp in https://github.com/dora-rs/dora/pull/453
-- Fix broken link in README by @mshr-h in https://github.com/dora-rs/dora/pull/462
-- fix cargo run --example cmake-dataflow compile bugs by @XxChang in https://github.com/dora-rs/dora/pull/460
-- Llm example by @haixuanTao in https://github.com/dora-rs/dora/pull/451
-- Fix meter conflict by @haixuanTao in https://github.com/dora-rs/dora/pull/461
-- Update README.md by @bobd988 in https://github.com/dora-rs/dora/pull/458
-- Refactor `README` by @haixuanTao in https://github.com/dora-rs/dora/pull/463
-- Specify conda env for Python Operators by @haixuanTao in https://github.com/dora-rs/dora/pull/468
-
-## Minor
-
-- Bump h2 from 0.3.24 to 0.3.26 by @dependabot in https://github.com/dora-rs/dora/pull/456
-- Update `bat` dependency to v0.24 by @phil-opp in https://github.com/dora-rs/dora/pull/424
-
-## New Contributors
-
-- @bobd988 made their first contribution in https://github.com/dora-rs/dora/pull/431
-
-* @mshr-h made their first contribution in https://github.com/dora-rs/dora/pull/462
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.2...v0.3.3
-
-## v0.3.2 (2024-01-26)
-
-## Features
-
-- Wait until `DestroyResult` is sent before exiting dora-daemon by @phil-opp in https://github.com/dora-rs/dora/pull/413
-- Reduce dora-rs to a single binary by @haixuanTao in https://github.com/dora-rs/dora/pull/410
-- Rework python ROS2 (de)serialization using parsed ROS2 messages directly by @phil-opp in https://github.com/dora-rs/dora/pull/415
-- Fix ros2 array bug by @haixuanTao in https://github.com/dora-rs/dora/pull/412
-- Test ros2 type info by @haixuanTao in https://github.com/dora-rs/dora/pull/418
-- Use forward slash as it is default way of defining ros2 topic by @haixuanTao in https://github.com/dora-rs/dora/pull/419
-
-## Minor
-
-- Bump h2 from 0.3.21 to 0.3.24 by @dependabot in https://github.com/dora-rs/dora/pull/414
-
-## v0.3.1 (2024-01-09)
-
-## Features
-
-- Support legacy python by @haixuanTao in https://github.com/dora-rs/dora/pull/382
-- Add an error catch in python `on_event` when using hot-reloading by @haixuanTao in https://github.com/dora-rs/dora/pull/372
-- add cmake example by @XxChang in https://github.com/dora-rs/dora/pull/381
-- Bump opentelemetry metrics to 0.21 by @haixuanTao in https://github.com/dora-rs/dora/pull/383
-- Trace send_output as it can be a big source of overhead for large messages by @haixuanTao in https://github.com/dora-rs/dora/pull/384
-- Adding a timeout method to not block indefinitely next event by @haixuanTao in https://github.com/dora-rs/dora/pull/386
-- Adding `Vec<u8>` conversion by @haixuanTao in https://github.com/dora-rs/dora/pull/387
-- Dora cli renaming by @haixuanTao in https://github.com/dora-rs/dora/pull/399
-- Update `ros2-client` and `rustdds` dependencies to latest fork version by @phil-opp in https://github.com/dora-rs/dora/pull/397
-
-## Fix
-
-- Fix window path error by @haixuanTao in https://github.com/dora-rs/dora/pull/398
-- Fix read error in C++ node input by @haixuanTao in https://github.com/dora-rs/dora/pull/406
-- Bump unsafe-libyaml from 0.2.9 to 0.2.10 by @dependabot in https://github.com/dora-rs/dora/pull/400
-
-## New Contributors
-
-- @XxChang made their first contribution in https://github.com/dora-rs/dora/pull/381
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.0...v0.3.1
-
-## v0.3.0 (2023-11-01)
-
-## Features
-
-- Rust node API typed using arrow by @phil-opp in https://github.com/dora-rs/dora/pull/353
-- Dora record by @haixuanTao in https://github.com/dora-rs/dora/pull/365
-- beautify graph visualisation by @haixuanTao in https://github.com/dora-rs/dora/pull/370
-- Remove `Ros2Value` encapsulation of `ArrayData` by @haixuanTao in https://github.com/dora-rs/dora/pull/359
-- Refactor python typing by @haixuanTao in https://github.com/dora-rs/dora/pull/369
-- Update README discord link by @Felixhuangsiling in https://github.com/dora-rs/dora/pull/361
-
-### Other
-
-- Update `rustix` v0.38 dependency by @phil-opp in https://github.com/dora-rs/dora/pull/366
-- Bump rustix from 0.37.24 to 0.37.25 by @dependabot in https://github.com/dora-rs/dora/pull/364
-- Bump quinn-proto from 0.9.3 to 0.9.5 by @dependabot in https://github.com/dora-rs/dora/pull/357
-- Bump webpki from 0.22.1 to 0.22.2 by @dependabot in https://github.com/dora-rs/dora/pull/358
-- Update README discord link by @Felixhuangsiling in https://github.com/dora-rs/dora/pull/361
-
-## New Contributors
-
-- @Felixhuangsiling made their first contribution in https://github.com/dora-rs/dora/pull/361
-
-## v0.2.6 (2023-09-14)
-
-- Update dependencies to fix some security advisories by @phil-opp in https://github.com/dora-rs/dora/pull/354
-  - Fixes `cargo install dora-daemon`
-
-## v0.2.5 (2023-09-06)
-
-### Features
-
-- Use cargo instead of git in Rust `Cargo.toml` template by @haixuanTao in https://github.com/dora-rs/dora/pull/326
-- Use read_line instead of next_line in stderr by @haixuanTao in https://github.com/dora-rs/dora/pull/325
-- Add a `rust-ros2-dataflow` example using the dora-ros2-bridge by @phil-opp in https://github.com/dora-rs/dora/pull/324
-- Removing patchelf by @haixuanTao in https://github.com/dora-rs/dora/pull/333
-- Improving python example readability by @haixuanTao in https://github.com/dora-rs/dora/pull/334
-- Use `serde_bytes` to serialize `Vec<u8>` by @haixuanTao in https://github.com/dora-rs/dora/pull/336
-- Adding support for `Arrow List(*)` for Python by @haixuanTao in https://github.com/dora-rs/dora/pull/337
-- Bump rustls-webpki from 0.100.1 to 0.100.2 by @dependabot in https://github.com/dora-rs/dora/pull/340
-- Add support for event stream merging for Python node API by @phil-opp in https://github.com/dora-rs/dora/pull/339
-- Merge `dora-ros2-bridge` by @phil-opp in https://github.com/dora-rs/dora/pull/341
-- Update dependencies by @phil-opp in https://github.com/dora-rs/dora/pull/345
-- Add support for arbitrary Arrow types in Python API by @phil-opp in https://github.com/dora-rs/dora/pull/343
-- Use typed inputs in Python ROS2 example by @phil-opp in https://github.com/dora-rs/dora/pull/346
-- Use struct type instead of array for ros2 messages by @haixuanTao in https://github.com/dora-rs/dora/pull/349
-
-### Other
-
-- Add Discord :speech_balloon: by @haixuanTao in https://github.com/dora-rs/dora/pull/348
-- Small refactoring by @haixuanTao in https://github.com/dora-rs/dora/pull/342
-
-## v0.2.4 (2023-07-18)
-
-### Features
-
-- Return dataflow result to CLI on `dora stop` by @phil-opp in https://github.com/dora-rs/dora/pull/300
-- Make dataflow descriptor available to Python nodes and operators by @phil-opp in https://github.com/dora-rs/dora/pull/301
-- Create a `CONTRIBUTING.md` guide by @phil-opp in https://github.com/dora-rs/dora/pull/307
-- Distribute prebuilt arm macos dora-rs by @haixuanTao in https://github.com/dora-rs/dora/pull/308
-
-### Other
-
-- Fix the typos and add dora code branch by @meua in https://github.com/dora-rs/dora/pull/290
-- For consistency with other examples, modify python -> python3 by @meua in https://github.com/dora-rs/dora/pull/299
-- Add timestamps generated by hybrid logical clocks to all sent events by @phil-opp in https://github.com/dora-rs/dora/pull/302
-- Don't recompile the `dora-operator-api-c` crate on every build/run by @phil-opp in https://github.com/dora-rs/dora/pull/304
-- Remove deprecated `proc_macros` feature from `safer-ffi` dependency by @phil-opp in https://github.com/dora-rs/dora/pull/305
-- Update to Rust v1.70 by @phil-opp in https://github.com/dora-rs/dora/pull/303
-- Fix issue with not finding a custom nodes path by @haixuanTao in https://github.com/dora-rs/dora/pull/315
-- Implement `Stream` for `EventStream` by @phil-opp in https://github.com/dora-rs/dora/pull/309
-- Replace unmaintained `atty` crate with `std::io::IsTerminal` by @phil-opp in https://github.com/dora-rs/dora/pull/318
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.2.3...v0.2.4
-
-## v0.2.3 (2023-05-24)
-
-## What's Changed
-
-- Check that coordinator, daemon, and node versions match by @phil-opp in https://github.com/dora-rs/dora/pull/245
-- Share events to Python without copying via `arrow` crate by @phil-opp in https://github.com/dora-rs/dora/pull/228
-- Upgrading the operator example to use `dora-arrow` by @haixuanTao in https://github.com/dora-rs/dora/pull/251
-- [Python] Show node name in process and put Traceback before the actual Error for more natural error by @haixuanTao in https://github.com/dora-rs/dora/pull/255
-- CLI: Improve error messages when coordinator is not running by @phil-opp in https://github.com/dora-rs/dora/pull/254
-- Integrate `dora-runtime` into `dora-daemon` by @phil-opp in https://github.com/dora-rs/dora/pull/257
-- Filter default log level at `warn` for `tokio::tracing` by @haixuanTao in https://github.com/dora-rs/dora/pull/269
-- Make log level filtering be `WARN` or below by @haixuanTao in https://github.com/dora-rs/dora/pull/274
-- Add support for distributed deployments with multiple daemons by @phil-opp in https://github.com/dora-rs/dora/pull/256
-- Provide a way to access logs through the CLI by @haixuanTao in https://github.com/dora-rs/dora/pull/259
-- Handle node errors during initialization phase by @phil-opp in https://github.com/dora-rs/dora/pull/275
-- Replace watchdog by asynchronous heartbeat messages by @phil-opp in https://github.com/dora-rs/dora/pull/278
-- Remove pyo3 in runtime and daemon as it generates `libpython` depende… by @haixuanTao in https://github.com/dora-rs/dora/pull/281
-- Release v0.2.3 with aarch64 support by @haixuanTao in https://github.com/dora-rs/dora/pull/279
-
-## Fix
-
-- Fix yolov5 dependency issue by @haixuanTao in https://github.com/dora-rs/dora/pull/291
-- To solve this bug https://github.com/dora-rs/dora/issues/283, unify t… by @meua in https://github.com/dora-rs/dora/pull/285
-- Fix: Don't try to create two global tracing subscribers when using bundled runtime by @phil-opp in https://github.com/dora-rs/dora/pull/277
-- CI: Increase timeout for 'build CLI and binaries' step by @phil-opp in https://github.com/dora-rs/dora/pull/282
-
-## Other
-
-- Update `pyo3` to `v0.18` by @phil-opp in https://github.com/dora-rs/dora/pull/246
-- Bump h2 from 0.3.13 to 0.3.17 by @dependabot in https://github.com/dora-rs/dora/pull/249
-- Add automatic issue labeler to organize opened issues by @haixuanTao in https://github.com/dora-rs/dora/pull/265
-- Allow the issue labeler to write issues by @phil-opp in https://github.com/dora-rs/dora/pull/272
-- Add a support matrix with planned feature to clarify dora status by @haixuanTao in https://github.com/dora-rs/dora/pull/264
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.2.2...v0.2.3
-
-## v0.2.2 (2023-04-01)
-
-### Features
-
-- Make queue length configurable through the dataflow file by @phil-opp in https://github.com/dora-rs/dora/pull/231
-- Hot reloading Python Operator by @haixuanTao in https://github.com/dora-rs/dora/pull/239
-- Synchronize node and operator start by @phil-opp in https://github.com/dora-rs/dora/pull/236
-- Add opentelemetry capability at runtime instead of compile time by @haixuanTao in https://github.com/dora-rs/dora/pull/234
-
-### Others
-
-- Wait on events and messages simultaneously to prevent queue buildup by @phil-opp in https://github.com/dora-rs/dora/pull/235
-- Fix looping in daemon listener loop by @phil-opp in https://github.com/dora-rs/dora/pull/244
-- Validate shell command as source and url source by @haixuanTao in https://github.com/dora-rs/dora/pull/243
-- Push error into the `init_done` channel for debugging context by @haixuanTao in https://github.com/dora-rs/dora/pull/238
-- Option communication config by @haixuanTao in https://github.com/dora-rs/dora/pull/241
-- Validate yaml when reading by @haixuanTao in https://github.com/dora-rs/dora/pull/237
-
-**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.2.1...v0.2.2
-
-## v0.2.1 (2023-03-22)
-
-### Features
-
-- [Make dora-rs publishable on crates.io](https://github.com/dora-rs/dora/pull/211)
-
-### Fixes
-
-- [Avoid blocking the daemon main loop by using unbounded queue](https://github.com/dora-rs/dora/pull/230)
-- [Inject YAML declared env variable into the runtime](https://github.com/dora-rs/dora/pull/227)
-- [Use rustls instead of system SSL implementation](https://github.com/dora-rs/dora/pull/216)
-
-### Other
-
-- [Refactor python error](https://github.com/dora-rs/dora/pull/229)
-- [The first letter of rust should be lowercase in the command](https://github.com/dora-rs/dora/pull/226)
-- [Add documentation to the cli within the helper mode](https://github.com/dora-rs/dora/pull/225)
-- [Update to safer-ffi v0.1.0-rc1](https://github.com/dora-rs/dora/pull/218)
-- [remove unused variable: data_bytes](https://github.com/dora-rs/dora/pull/215)
-- [Clean up: Remove workspace path](https://github.com/dora-rs/dora/pull/210)
-- [Decouple opentelemetry from tracing](https://github.com/dora-rs/dora/pull/222)
-- [Remove zenoh dependency from dora node API to speed up build](https://github.com/dora-rs/dora/pull/220)
-- [Update to Rust v1.68](https://github.com/dora-rs/dora/pull/221)
-- [Deny unknown fields to avoid typos](https://github.com/dora-rs/dora/pull/223)
-- [Add an internal cli argument to create template with path dependencies](https://github.com/dora-rs/dora/pull/212)
-
-## v0.2.0 (2023-03-14)
-
-### Breaking
-
-- [Redesign: Create a `dora-daemon` as a communication broker](https://github.com/dora-rs/dora/pull/162)
-  - New `dora-daemon` executable that acts as a communication hub for all local nodes
-  - Large messages are passed through shared memory without any copying
-  - [Replaces the previous `iceoryx` communication layer](https://github.com/dora-rs/dora/pull/201)
-  - Small API change: Nodes and operators now receive _events_ instead of just inputs
-    - Inputs are one type of event
-    - Other supported events: `InputClosed` when an input stream is closed and `Stop` when the user stops the dataflow (e.g. through the CLI)
-
-### Features
-
-- Better Error handling when operator fails
-- [Send small messages directly without shared memory](https://github.com/dora-rs/dora/pull/193)
-- [Send all queued incoming events at once on `NextEvent` request](https://github.com/dora-rs/dora/pull/194)
-- [Don't send replies for `SendMessage` requests when using TCP](https://github.com/dora-rs/dora/pull/195)
-- [Allocate shared memory in nodes to improve throughput](https://github.com/dora-rs/dora/pull/200)
-
-### Fixes
-
-- [Manage node failure: Await all nodes to finish before marking dataflow as finished](https://github.com/dora-rs/dora/pull/183)
-
-### Other
-
-- [Use `DoraStatus` from dora library in template](https://github.com/dora-rs/dora/pull/182)
-- [Simplify: Replace `library_filename` function with `format!` call](https://github.com/dora-rs/dora/pull/191)
-- [Refactor Rust node API implementation](https://github.com/dora-rs/dora/pull/196)
-- [Remove code duplicate for tracing subscriber and use env variable to manage log level.](https://github.com/dora-rs/dora/pull/197)
-- [Add daemon to the release archive](https://github.com/dora-rs/dora/pull/199)
-- [Remove `remove_dir_all` from `Cargo.lock`as it is vulnerable to a race condition according to dependabot](https://github.com/dora-rs/dora/pull/202)
-- [Update the documentation to the new daemon format](https://github.com/dora-rs/dora/pull/198)
-- [Removing legacy `libacl` which was required by Iceoryx](https://github.com/dora-rs/dora/pull/205)
-- [Remove unimplemented CLI arguments for now](https://github.com/dora-rs/dora/pull/207)
-- [Update zenoh to remove git dependencies](https://github.com/dora-rs/dora/pull/203)
-- [Fix cli template to new daemon API](https://github.com/dora-rs/dora/pull/204)
-- [Cleanup warnings](https://github.com/dora-rs/dora/pull/208)
-- Dependency updates
-
-## v0.1.3 (2023-01-18)
-
-- Package `DoraStatus` into dora python package: https://github.com/dora-rs/dora/pull/172
-- Force removal of Pyo3 Object to avoid memory leak: https://github.com/dora-rs/dora/pull/168
-- Bump tokio from 1.21.2 to 1.23.1: https://github.com/dora-rs/dora/pull/171
-- Create a changelog file: https://github.com/dora-rs/dora/pull/174
-
-## v0.1.2 (2022-12-15)
-
-- Fix infinite loop in the coordinator: https://github.com/dora-rs/dora/pull/155
-- Simplify the release process: https://github.com/dora-rs/dora/pull/157
-- Use generic linux distribution: https://github.com/dora-rs/dora/pull/159
-
-## v0.1.1 (2022-12-05)
-
-This release contains fixes for:
-
-- Python linking using pypi release but also a redesigned python thread model within the runtime to avoid deadlock of the `GIL`. This also fix an issue with `patchelf`.
-- A deployment separation for `ubuntu` as the `20.04` version of `dora` and `22.04` version of dora are non-compatible.
-- A better tagging of api for `dora` Rust API.
-
-## v0.1.0 (2022-11-15)
-
-This is our first release of `dora-rs`!
-
-The current release includes:
-
-- `dora-cli` which enables creating, starting and stopping dataflow.
-- `dora-coordinator` which is our control plane.
-- `dora-runtime` which is manage the runtime of operators.
-- `custom-nodes` API which enables bridges from different languages.

NOTICE.md

@@ -1,7 +0,0 @@
-## Copyright
-
-All content is the property of the respective authors or their employers. For more information regarding authorship of content, please consult the listed source code repository logs.
-
-## License
-
-This project is licensed under the Apache License, Version 2.0 ([LICENSE](LICENSE) or <http://www.apache.org/licenses/LICENSE-2.0>). Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be licensed as above, without any additional terms or conditions.

README.md

@@ -1,359 +0,0 @@
-#
-
-<p align="center">
-    <img src="https://raw.githubusercontent.com/dora-rs/dora/main/docs/src/logo.svg" width="400"/>
-</p>
-
-<h2 align="center">
-  <a href="https://www.dora-rs.ai">Website</a>
-  |
-  <a href="https://dora-rs.ai/docs/guides/getting-started/conversation_py/">Python API</a>
-  |
-  <a href="https://docs.rs/dora-node-api/latest/dora_node_api/">Rust API</a>
-  |
-  <a href="https://www.dora-rs.ai/docs/guides/">Guide</a>
-  |
-  <a href="https://discord.gg/6eMGGutkfE">Discord</a>
-</h2>
-
-<div align="center">
-  <a href="https://github.com/dora-rs/dora/actions">
-    <img src="https://github.com/dora-rs/dora/workflows/CI/badge.svg" alt="Build and test"/>
-  </a>
-  <a href="https://crates.io/crates/dora-rs">
-    <img src="https://img.shields.io/crates/v/dora_node_api.svg"/>
-  </a>
-  <a href="https://docs.rs/dora-node-api/latest/dora_node_api/">
-    <img src="https://docs.rs/dora-node-api/badge.svg" alt="rust docs"/>
-  </a>
-  <a href="https://pypi.org/project/dora-rs/">
-    <img src="https://img.shields.io/pypi/v/dora-rs.svg" alt="PyPi Latest Release"/>
-  </a>
-    <a href="https://github.com/dora-rs/dora/blob/main/LICENSE">
-    <img src="https://img.shields.io/github/license/dora-rs/dora" alt="PyPi Latest Release"/>
-  </a>      
-</div>
-<div align="center">
-<a href="https://trendshift.io/repositories/9190" target="_blank"><img src="https://trendshift.io/api/badge/repositories/9190" alt="dora-rs%2Fdora | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
-</div>
-
-## Highlights
-
-- 🚀 dora-rs is a framework to run realtime multi-AI and multi-hardware applications.
-- 🦀 dora-rs internals are 100% Rust making it extremely fast compared to alternative such as being ⚡️ [10-17x faster](https://github.com/dora-rs/dora-benchmark) than `ros2`.
-- ❇️ Includes a large set of pre-packaged nodes for fast prototyping which simplifies integration of hardware, algorithms, and AI models.
-
-<p align="center">
-  <picture align="center">
-    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/dora-rs/dora/main/docs/src/bar_chart_dark.svg">
-    <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/dora-rs/dora/main/docs/src/bar_chart_light.svg">
-    <img src="https://raw.githubusercontent.com/dora-rs/dora/main/docs/src/bar_chart_light.svg">
-  </picture>
-</p>
-
-<p align="center">
- <a href="https://github.com/dora-rs/dora-benchmark/" >
-  <i>Latency benchmark with Python API for both framework, sending 40M of random bytes.</i>
-  </a>
-</p>
-
-## Latest News 🎉
-
-<details open>
-<summary><b>2025</b></summary>
-
-- \[07/25\] Added Kornia rust nodes in the hub for V4L / Gstreamer cameras and Sobel image processing.
-- \[06/25\] Add support for git based node, dora-vggt for multi-camera depth estimation, and adding robot_descriptions_py as a default way to get urdfs within dora.
-- \[05/25\] Add support for dora-pytorch-kinematics for fk and ik, dora-mediapipe for pose estimation, dora-rustypot for rust serialport read/write, points2d and points3d visualization in rerun.
-- \[04/25\] Add support for dora-cotracker to track any point on a frame, dora-rav1e AV1 encoding up to 12bit and dora-dav1d AV1 decoding,
-- \[03/25\] Add support for dora async Python.
-- \[03/25\] Add support for Microsoft Phi4, Microsoft Magma.
-- \[03/25\] dora-rs has been accepted to [**GSoC 2025 🎉**](https://summerofcode.withgoogle.com/programs/2025/organizations/dora-rs-tb), with the following [**idea list**](https://github.com/dora-rs/dora/wiki/GSoC_2025).
-- \[03/25\] Add support for Zenoh for distributed dataflow.
-- \[03/25\] Add support for Meta SAM2, Kokoro(TTS), Improved Qwen2.5 Performance using `llama.cpp`.
-- \[02/25\] Add support for Qwen2.5(LLM), Qwen2.5-VL(VLM), outetts(TTS)
-</details>
-
-## Support Matrix
-
-|                                   | dora-rs                                                                                                                                                                                                     |
-| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **APIs**                          | Python >= 3.7 including sync ⭐✅ <br> Rust ✅<br> C/C++ 🆗 <br>ROS2 >= Foxy 🆗                                                                                                                             |
-| **OS**                            | Linux: Arm 32 ⭐✅ Arm 64 ⭐✅ x64_86 ⭐✅ <br>MacOS: Arm 64 ⭐✅ <br>Windows: x64_86 🆗 <br>WSL: x64_86 🆗 <br> Android: 🛠️ (Blocked by: https://github.com/elast0ny/shared_memory/issues/32) <br> IOS: 🛠️ |
-| **Message Format**                | Arrow ✅ <br> Standard Specification 🛠️                                                                                                                                                                     |
-| **Local Communication**           | Shared Memory ✅ <br> [Cuda IPC](https://arrow.apache.org/docs/python/api/cuda.html) 📐                                                                                                                     |
-| **Remote Communication**          | [Zenoh](https://zenoh.io/) 📐                                                                                                                                                                               |
-| **Metrics, Tracing, and Logging** | Opentelemetry 📐                                                                                                                                                                                            |
-| **Configuration**                 | YAML ✅                                                                                                                                                                                                     |
-| **Package Manager**               | [pip](https://pypi.org/): Python Node ✅ Rust Node ✅ C/C++ Node 🛠️ <br>[cargo](https://crates.io/): Rust Node ✅                                                                                           |
-
-> - ⭐ = Recommended
-> - ✅ = First Class Support
-> - 🆗 = Best Effort Support
-> - 📐 = Experimental and looking for contributions
-> - 🛠️ = Unsupported but hoped for through contributions
->
-> Everything is open for contributions 🙋
-
-## Node Hub
-
-> Feel free to modify this README with your own nodes so that it benefits the community.
-
-| Type                          | Title                                                                                               | Support             | Description                                      | Downloads                                                                     | License                                                                    |
-| ----------------------------- | --------------------------------------------------------------------------------------------------- | ------------------- | ------------------------------------------------ | ----------------------------------------------------------------------------- | -------------------------------------------------------------------------- |
-| Camera                        | [PyOrbbeckSDK](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyorbbecksdk)                | 📐                  | Image and depth from Orbbeck Camera              | ![Downloads](https://img.shields.io/pypi/dm/dora-pyorbbecksdk?label=%20)      | ![License](https://img.shields.io/pypi/l/dora-pyorbbecksdk?label=%20)      |
-| Camera                        | [PyRealsense](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyrealsense)                  | Linux🆗 <br> Mac🛠️  | Image and depth from Realsense                   | ![Downloads](https://img.shields.io/pypi/dm/dora-pyrealsense?label=%20)       | ![License](https://img.shields.io/pypi/l/dora-pyrealsense?label=%20)       |
-| Camera                        | [OpenCV Video Capture](https://github.com/dora-rs/dora/blob/main/node-hub/opencv-video-capture)     | ✅                  | Image stream from OpenCV Camera                  | ![Downloads](https://img.shields.io/pypi/dm/opencv-video-capture?label=%20)   | ![License](https://img.shields.io/pypi/l/opencv-video-capture?label=%20)   |
-| Camera                        | [Kornia V4L Capture](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-v4l-capture)         | ✅                  | Video stream for Linux Camera (rust)             |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)           |
-| Camera                        | [Kornia GST Capture](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-gst-capture)         | ✅                  | Video Capture using Gstreamer (rust)             |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)           |
-| Peripheral                    | [Keyboard](https://github.com/dora-rs/dora/blob/main/node-hub/dora-keyboard)                        | ✅                  | Keyboard char listener                           | ![Downloads](https://img.shields.io/pypi/dm/dora-keyboard?label=%20)          | ![License](https://img.shields.io/pypi/l/dora-keyboard?label=%20)          |
-| Peripheral                    | [Microphone](https://github.com/dora-rs/dora/blob/main/node-hub/dora-microphone)                    | ✅                  | Audio from microphone                            | ![Downloads](https://img.shields.io/pypi/dm/dora-microphone?label=%20)        | ![License](https://img.shields.io/pypi/l/dora-microphone?label=%20)        |
-| Peripheral                    | [PyAudio(Speaker)](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyaudio)                 | ✅                  | Output audio from speaker                        | ![Downloads](https://img.shields.io/pypi/dm/dora-pyaudio?label=%20)           | ![License](https://img.shields.io/pypi/l/dora-pyaudio?label=%20)           |
-| Actuator                      | [Feetech](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/feetech-client)                | 📐                  | Feetech Client                                   |                                                                               |                                                                            |
-| Actuator                      | [Dynamixel](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/dynamixel-client)            | 📐                  | Dynamixel Client                                 |                                                                               |                                                                            |
-| Chassis                       | [Agilex - UGV](https://github.com/dora-rs/dora/blob/main/node-hub/dora-ugv)                         | 🆗                  | Robomaster Client                                | ![Downloads](https://img.shields.io/pypi/dm/dora-ugv?label=%20)               | ![License](https://img.shields.io/pypi/l/dora-ugv?label=%20)               |
-| Chassis                       | [DJI - Robomaster S1](https://huggingface.co/datasets/dora-rs/dora-robomaster)                      | 📐                  | Robomaster Client                                |                                                                               |                                                                            |
-| Chassis                       | [Dora Kit Car](https://github.com/dora-rs/dora/blob/main/node-hub/dora-kit-car)                     | 🆗                  | Open Source Chassis                              | ![Downloads](https://img.shields.io/pypi/dm/dora-kit-car?label=%20)           | ![License](https://img.shields.io/pypi/l/dora-kit-car?label=%20)           |
-| Arm                           | [Alex Koch - Low Cost Robot](https://github.com/dora-rs/dora-lerobot/blob/main/robots/alexk-lcr)    | 📐                  | Alex Koch - Low Cost Robot Client                |                                                                               |                                                                            |
-| Arm                           | [Lebai - LM3](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/lebai-client)              | 📐                  | Lebai client                                     |                                                                               |                                                                            |
-| Arm                           | [Agilex - Piper](https://github.com/dora-rs/dora/blob/main/node-hub/dora-piper)                     | 🆗                  | Agilex arm client                                | ![Downloads](https://img.shields.io/pypi/dm/dora-piper?label=%20)             | ![License](https://img.shields.io/pypi/l/dora-piper?label=%20)             |
-| Robot                         | [Pollen - Reachy 1](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/dora-reachy1)        | 📐                  | Reachy 1 Client                                  |                                                                               |                                                                            |
-| Robot                         | [Pollen - Reachy 2](https://github.com/dora-rs/dora/blob/main/node-hub/dora-reachy2)                | 🆗                  | Reachy 2 client                                  | ![Downloads](https://img.shields.io/pypi/dm/dora-reachy2?label=%20)           | ![License](https://img.shields.io/pypi/l/dora-reachy2?label=%20)           |
-| Robot                         | [Trossen - Aloha](https://github.com/dora-rs/dora-lerobot/blob/main/robots/aloha)                   | 📐                  | Aloha client                                     |                                                                               |                                                                            |
-| Voice Activity Detection(VAD) | [Silero VAD](https://github.com/dora-rs/dora/blob/main/node-hub/dora-vad)                           | ✅                  | Silero Voice activity detection                  | ![Downloads](https://img.shields.io/pypi/dm/dora-vad?label=%20)               | ![License](https://img.shields.io/pypi/l/dora-vad?label=%20)               |
-| Speech to Text(STT)           | [Whisper](https://github.com/dora-rs/dora/blob/main/node-hub/dora-distil-whisper)                   | ✅                  | Transcribe audio to text                         | ![Downloads](https://img.shields.io/pypi/dm/dora-distil-whisper?label=%20)    | ![License](https://img.shields.io/pypi/l/dora-distil-whisper?label=%20)    |
-| Object Detection              | [Yolov8](https://github.com/dora-rs/dora/blob/main/node-hub/dora-yolo)                              | ✅                  | Object detection                                 | ![Downloads](https://img.shields.io/pypi/dm/dora-yolo?label=%20)              | ![License](https://img.shields.io/pypi/l/dora-yolo?label=%20)              |
-| Segmentation                  | [SAM2](https://github.com/dora-rs/dora/blob/main/node-hub/dora-sam2)                                | Cuda✅ <br> Metal🛠️ | Segment Anything                                 | ![Downloads](https://img.shields.io/pypi/dm/dora-sam2?label=%20)              | ![License](https://img.shields.io/pypi/l/dora-sam2?label=%20)              |
-| Large Language Model(LLM)     | [Qwen2.5](https://github.com/dora-rs/dora/blob/main/node-hub/dora-qwen)                             | ✅                  | Large Language Model using Qwen                  | ![Downloads](https://img.shields.io/pypi/dm/dora-qwen?label=%20)              | ![License](https://img.shields.io/pypi/l/dora-qwen?label=%20)              |
-| Vision Language Model(VLM)    | [Qwen2.5-vl](https://github.com/dora-rs/dora/blob/main/node-hub/dora-qwen2-5-vl)                    | ✅                  | Vision Language Model using Qwen2.5 VL           | ![Downloads](https://img.shields.io/pypi/dm/dora-qwen2-5-vl?label=%20)        | ![License](https://img.shields.io/pypi/l/dora-qwen2-5-vl?label=%20)        |
-| Vision Language Model(VLM)    | [InternVL](https://github.com/dora-rs/dora/blob/main/node-hub/dora-internvl)                        | 🆗                  | InternVL is a vision language model              | ![Downloads](https://img.shields.io/pypi/dm/dora-internvl?label=%20)          | ![License](https://img.shields.io/pypi/l/dora-internvl?label=%20)          |
-| Vision Language Action(VLA)   | [RDT-1B](https://github.com/dora-rs/dora/blob/main/node-hub/dora-rdt-1b)                            | 🆗                  | Infer policy using Robotic Diffusion Transformer | ![Downloads](https://img.shields.io/pypi/dm/dora-rdt-1b?label=%20)            | ![License](https://img.shields.io/pypi/l/dora-rdt-1b?label=%20)            |
-| Translation                   | [ArgosTranslate](https://github.com/dora-rs/dora/blob/main/node-hub/dora-argotranslate)             | 🆗                  | Open Source translation engine                   | ![Downloads](https://img.shields.io/pypi/dm/dora-argotranslate?label=%20)     | ![License](https://img.shields.io/pypi/l/dora-argotranslate?label=%20)     |
-| Translation                   | [Opus MT](https://github.com/dora-rs/dora/blob/main/node-hub/dora-opus)                             | 🆗                  | Translate text between language                  | ![Downloads](https://img.shields.io/pypi/dm/dora-opus?label=%20)              | ![License](https://img.shields.io/pypi/l/dora-opus?label=%20)              |
-| Text to Speech(TTS)           | [Kokoro TTS](https://github.com/dora-rs/dora/blob/main/node-hub/dora-kokoro-tts)                    | ✅                  | Efficient Text to Speech                         | ![Downloads](https://img.shields.io/pypi/dm/dora-kokoro-tts?label=%20)        | ![License](https://img.shields.io/pypi/l/dora-kokoro-tts?label=%20)        |
-| Recorder                      | [Llama Factory Recorder](https://github.com/dora-rs/dora/blob/main/node-hub/llama-factory-recorder) | 🆗                  | Record data to train LLM and VLM                 | ![Downloads](https://img.shields.io/pypi/dm/llama-factory-recorder?label=%20) | ![License](https://img.shields.io/pypi/l/llama-factory-recorder?label=%20) |
-| Recorder                      | [LeRobot Recorder](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/lerobot-dashboard)    | 📐                  | LeRobot Recorder helper                          |                                                                               |                                                                            |
-| Visualization                 | [Plot](https://github.com/dora-rs/dora/blob/main/node-hub/opencv-plot)                              | ✅                  | Simple OpenCV plot visualization                 | ![Downloads](https://img.shields.io/pypi/dm/dora-yolo?label=%20)              | ![License](https://img.shields.io/pypi/l/opencv-plot?label=%20)            |
-| Visualization                 | [Rerun](https://github.com/dora-rs/dora/blob/main/node-hub/dora-rerun)                              | ✅                  | Visualization tool                               | ![Downloads](https://img.shields.io/pypi/dm/dora-rerun?label=%20)             | ![License](https://img.shields.io/pypi/l/dora-rerun?label=%20)             |
-| Simulator                     | [Mujoco](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/mujoco-client)                  | 📐                  | Mujoco Simulator                                 |                                                                               |                                                                            |
-| Simulator                     | [Carla](https://github.com/dora-rs/dora-drives)                                                     | 📐                  | Carla Simulator                                  |                                                                               |                                                                            |
-| Simulator                     | [Gymnasium](https://github.com/dora-rs/dora-lerobot/blob/main/gym_dora)                             | 📐                  | Experimental OpenAI Gymnasium bridge             |                                                                               |                                                                            |
-| Image Processing              | [Kornia Sobel Operator](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-imgproc-sobel)    | ✅                  | Kornia image processing Sobel operator (rust)    |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)           |
-
-## Examples
-
-| Type           | Title                                                                                                        | Description                                         | Last Commit                                                                                                        |
-| -------------- | ------------------------------------------------------------------------------------------------------------ | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ |
-| Audio          | [Speech to Text(STT)](https://github.com/dora-rs/dora/blob/main/examples/speech-to-text)                     | Transform speech to text.                           | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fspeech-to-text&label=%20)        |
-| Audio          | [Translation](https://github.com/dora-rs/dora/blob/main/examples/translation)                                | Translate audio in real time.                       | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Ftranslation&label=%20)           |
-| Vision         | [Vision Language Model(VLM)](https://github.com/dora-rs/dora/blob/main/examples/vlm)                         | Use a VLM to understand images.                     | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fvlm&label=%20)                   |
-| Vision         | [YOLO](https://github.com/dora-rs/dora/blob/main/examples/python-dataflow)                                   | Use YOLO to detect object within image.             | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpython-dataflow&label=%20)       |
-| Vision         | [Camera](https://github.com/dora-rs/dora/blob/main/examples/camera)                                          | Simple webcam plot example                          | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fcamera&label=%20)                |
-| Vision         | [Image Processing](https://github.com/kornia/kornia-rs/tree/main/examples/dora)                              | Multi camera image processing                       |                                                                                                                    |
-| Model Training | [Piper RDT](https://github.com/dora-rs/dora/blob/main/examples/piper)                                        | Piper RDT Pipeline                                  | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpiper&label=%20)                 |
-| Model Training | [LeRobot - Alexander Koch](https://raw.githubusercontent.com/dora-rs/dora-lerobot/refs/heads/main/README.md) | Training Alexander Koch Low Cost Robot with LeRobot | ![License](https://img.shields.io/github/last-commit/dora-rs/dora-lerobot?path=robots&label=%20)                   |
-| ROS2           | [C++ ROS2 Example](https://github.com/dora-rs/dora/blob/main/examples/c++-ros2-dataflow)                     | Example using C++ ROS2                              | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fc%2b%2b-ros2-dataflow&label=%20) |
-| ROS2           | [Rust ROS2 Example](https://github.com/dora-rs/dora/blob/main/examples/rust-ros2-dataflow)                   | Example using Rust ROS2                             | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Frust-ros2-dataflow&label=%20)    |
-| ROS2           | [Python ROS2 Example](https://github.com/dora-rs/dora/blob/main/examples/python-ros2-dataflow)               | Example using Python ROS2                           | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpython-ros2-dataflow&label=%20)  |
-| Benchmark      | [GPU Benchmark](https://github.com/dora-rs/dora/blob/main/examples/cuda-benchmark)                           | GPU Benchmark of dora-rs                            | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fcuda-benchmark&label=%20)        |
-| Benchmark      | [CPU Benchmark](https://github.com/dora-rs/dora-benchmark/blob/main)                                         | CPU Benchmark of dora-rs                            | ![License](https://img.shields.io/github/last-commit/dora-rs/dora-benchmark?path=dora-rs&label=%20)                |
-| Tutorial       | [Rust Example](https://github.com/dora-rs/dora/blob/main/examples/rust-dataflow)                             | Example using Rust                                  | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Frust-dataflow&label=%20)         |
-| Tutorial       | [Python Example](https://github.com/dora-rs/dora/blob/main/examples/python-dataflow)                         | Example using Python                                | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpython-dataflow&label=%20)       |
-| Tutorial       | [CMake Example](https://github.com/dora-rs/dora/blob/main/examples/cmake-dataflow)                           | Example using CMake                                 | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fcmake-dataflow&label=%20)        |
-| Tutorial       | [C Example](https://github.com/dora-rs/dora/blob/main/examples/c-dataflow)                                   | Example with C node                                 | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fc-dataflow&label=%20)            |
-| Tutorial       | [CUDA Example](https://github.com/dora-rs/dora/blob/main/examples/cuda-benchmark)                            | Example using CUDA Zero Copy                        | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fcuda-benchmark&label=%20)        |
-| Tutorial       | [C++ Example](https://github.com/dora-rs/dora/blob/main/examples/c++-dataflow)                               | Example with C++ node                               | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fc%2b%2b-dataflow&label=%20)      |
-
-## Getting Started
-
-### Installation
-
-```bash
-pip install dora-rs-cli
-```
-
-<details close>
-<summary><b>Additional installation methods</b></summary>
-
-Install dora with our standalone installers, or from [crates.io](https://crates.io/crates/dora-cli):
-
-### With cargo
-
-```bash
-cargo install dora-cli
-```
-
-### With Github release for macOS and Linux
-
-```bash
-curl --proto '=https' --tlsv1.2 -LsSf https://github.com/dora-rs/dora/releases/latest/download/dora-cli-installer.sh | sh
-```
-
-### With Github release for Windows
-
-```powershell
-powershell -ExecutionPolicy ByPass -c "irm https://github.com/dora-rs/dorareleases/latest/download/dora-cli-installer.ps1 | iex"
-```
-
-### With Source
-
-```bash
-git clone https://github.com/dora-rs/dora.git
-cd dora
-cargo build --release -p dora-cli
-PATH=$PATH:$(pwd)/target/release
-```
-
-</details>
-
-### Run
-
-- Run the yolo python example:
-
-```bash
-## Create a virtual environment
-uv venv --seed -p 3.11
-
-## Install nodes dependencies of a remote graph
-dora build https://raw.githubusercontent.com/dora-rs/dora/refs/heads/main/examples/object-detection/yolo.yml --uv
-
-## Run yolo graph
-dora run yolo.yml --uv
-```
-
-> Make sure to have a webcam
-
-To stop your dataflow, you can use <kbd>ctrl</kbd>+<kbd>c</kbd>
-
-- To understand what is happening, you can look at the dataflow with:
-
-```bash
-cat yolo.yml
-```
-
-- Resulting in:
-
-```yaml
-nodes:
-  - id: camera
-    build: pip install opencv-video-capture
-    path: opencv-video-capture
-    inputs:
-      tick: dora/timer/millis/20
-    outputs:
-      - image
-    env:
-      CAPTURE_PATH: 0
-      IMAGE_WIDTH: 640
-      IMAGE_HEIGHT: 480
-
-  - id: object-detection
-    build: pip install dora-yolo
-    path: dora-yolo
-    inputs:
-      image: camera/image
-    outputs:
-      - bbox
-
-  - id: plot
-    build: pip install dora-rerun
-    path: dora-rerun
-    inputs:
-      image: camera/image
-      boxes2d: object-detection/bbox
-```
-
-- In the above example, we can understand that the camera is sending image to both the rerun viewer as well as a yolo model that generates bounding box that is visualized within rerun.
-
-### Documentation
-
-The full documentation is available on [our website](https://dora-rs.ai/).
-A lot of guides are available on [this section](https://dora-rs.ai/docs/guides/) of our website.
-
-## What is Dora? And what features does Dora offer?
-
-**D**ataflow-**O**riented **R**obotic **A**rchitecture (`dora-rs`) is a framework that makes creation of robotic applications fast and simple.
-
-`dora-rs` implements a declarative dataflow paradigm where tasks are split between nodes isolated as individual processes.
-
-The dataflow paradigm has the advantage of creating an abstraction layer that makes robotic applications modular and easily configurable.
-
-### TCP Communication and Shared Memory
-
-Communication between nodes is handled with shared memory on a same machine and TCP on distributed machines. Our shared memory implementation tracks messages across processes and discards them when obsolete. Shared memory slots are cached to avoid new memory allocation.
-
-### Arrow Message Format
-
-Nodes communicate with Apache Arrow Data Format.
-
-[Apache Arrow](https://github.com/apache/arrow-rs) is a universal memory format for flat and hierarchical data. The Arrow memory format supports zero-copy reads for lightning-fast data access without serialization overhead. It defines a C data interface without any build-time or link-time dependency requirement, that means that `dora-rs` has **no compilation step** beyond the native compiler of your favourite language.
-
-### Opentelemetry
-
-dora-rs uses Opentelemetry to record all your logs, metrics and traces. This means that the data and telemetry can be linked using a shared abstraction.
-
-[Opentelemetry](https://opentelemetry.io/) is an open source observability standard that makes dora-rs telemetry collectable by most backends such as elasticsearch, prometheus, Datadog...
-
-Opentelemetry is language independent, backend agnostic, and easily collect distributed data, making it perfect for dora-rs applications.
-
-### ROS2 Bridge
-
-**Note**: this feature is marked as unstable.
-
-- Compilation Free Message passing to ROS 2
-- Automatic conversion ROS 2 Message <-> Arrow Array
-
-```python
-import pyarrow as pa
-
-# Configuration Boilerplate...
-turtle_twist_writer = ...
-
-## Arrow Based ROS2 Twist Message
-## which does not require ROS2 import
-message = pa.array([{
-            "linear": {
-                "x": 1,
-            },
-            "angular": {
-                "z": 1
-            },
-        }])
-
-turtle_twist_writer.publish(message)
-```
-
-> You might want to use ChatGPT to write the Arrow Formatting: https://chat.openai.com/share/4eec1c6d-dbd2-46dc-b6cd-310d2895ba15
-
-## Contributing
-
-We are passionate about supporting contributors of all levels of experience and would love to see
-you get involved in the project. See the
-[contributing guide](https://github.com/dora-rs/dora/blob/main/CONTRIBUTING.md) to get started.
-
-## Discussions
-
-Our main communication channels are:
-
-- [Our Discord server](https://discord.gg/6eMGGutkfE)
-- [Our Github Project Discussion](https://github.com/orgs/dora-rs/discussions)
-
-Feel free to reach out on any topic, issues or ideas.
-
-We also have [a contributing guide](CONTRIBUTING.md).
-
-## License
-
-This project is licensed under Apache-2.0. Check out [NOTICE.md](NOTICE.md) for more information.
-
----
-
-## Further Resources 📚
-
-- [Zenoh Documentation](https://zenoh.io/docs/getting-started/first-app/)
-- [DORA Zenoh Discussion (GitHub Issue #512)](https://github.com/dora-rs/dora/issues/512)
-- [Dora Autoware Localization Demo](https://github.com/dora-rs/dora-autoware-localization-demo)
-
-```
-
-```

_typos.toml

@@ -1,4 +0,0 @@
-[default.extend-identifiers]
-# *sigh* this just isn't worth the cost of fixing
-DeviceNDArray = "DeviceNDArray"
-Feedforward_2nd_Gain = "Feedforward_2nd_Gain"

apis/c++/node/Cargo.toml

@@ -1,44 +0,0 @@
-[package]
-name = "dora-node-api-cxx"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-crate-type = ["staticlib"]
-
-[features]
-default = ["tracing"]
-tracing = ["dora-node-api/tracing"]
-ros2-bridge = [
-    "dep:dora-ros2-bridge",
-    "dep:dora-ros2-bridge-msg-gen",
-    "dep:rust-format",
-    "dep:prettyplease",
-    "dep:serde",
-    "dep:serde-big-array",
-]
-
-[dependencies]
-cxx = "1.0.73"
-dora-node-api = { workspace = true }
-eyre = "0.6.8"
-dora-ros2-bridge = { workspace = true, optional = true }
-futures-lite = { version = "2.2" }
-serde = { version = "1.0.164", features = ["derive"], optional = true }
-serde-big-array = { version = "0.5.1", optional = true }
-arrow = { workspace = true, features = ["ffi"] }
-
-[build-dependencies]
-cxx-build = "1.0.73"
-dora-ros2-bridge-msg-gen = { workspace = true, optional = true }
-rust-format = { version = "0.3.4", features = [
-    "pretty_please",
-], optional = true }
-prettyplease = { version = "0.1", features = ["verbatim"], optional = true }

apis/c++/node/README.md

@@ -1,321 +0,0 @@
-# Dora Node API for C++
-
-Dora supports nodes written in C++ through this API crate.
-
-## Build
-
-- Clone the `dora` repository:
-  ```bash
-  > git clone https://github.com/dora-rs/dora.git
-  > cd dora
-  ```
-- Build the `dora-node-api-cxx` package:
-  ```bash
-  cargo build --package dora-node-api-cxx
-  ```
-  - This will result in `dora-node-api.h` and `dora-node-api.cc` files in the `target/cxxbridge/dora-node-api-cxx` directory.
-- Include the `dora-node-api.h` header file in your source file.
-- Add the `dora-node-api.cc` file to your compile and link steps.
-
-### Build with ROS2 Bridge
-
-Dora features an experimental ROS2 Bridge that enables dora nodes to publish and subscribe to ROS2 topics.
-To enable the bridge, use these steps:
-
-- Clone the `dora` repository (see above).
-- Source the ROS2 setup files (see [ROS2 docs](https://docs.ros.org/en/rolling/Tutorials/Beginner-CLI-Tools/Configuring-ROS2-Environment.html#source-the-setup-files))
-- Optional: Source package-specific ROS2 setup files if you want to use custom package-specific ROS2 messages in the bridge (see [ROS2 docs](https://docs.ros.org/en/rolling/Tutorials/Beginner-Client-Libraries/Creating-Your-First-ROS2-Package.html#source-the-setup-file))
-- Build the `dora-node-api-cxx` package **with the `ros2-bridge` feature enabled**:
-  ```bash
-  cargo build --package dora-node-api-cxx --features ros2-bridge
-  ```
-  - In addition to the `dora-node-api.h` and `dora-node-api.cc` files, this will place a `dora-ros2-bindings.h` and a `dora-ros2-bindings.cc` file in the `target/cxxbridge/dora-node-api-cxx` directory.
-- Include both the `dora-node-api.h` and the `dora-ros2-bindings.h` header files in your source file.
-- Add the `dora-node-api.cc` and `dora-ros2-bindings.cc` files to your compile and link steps.
-
-## Usage
-
-The `dora-node-api.h` header provides various functions to interact with Dora.
-
-### Init Dora Node
-
-All nodes need to register themselves with Dora at startup.
-To do that, call the `init_dora_node()` function.
-The function returns a `DoraNode` instance, which gives access to dora events and enables sending Dora outputs.
-
-```c++
-auto dora_node = init_dora_node();
-```
-
-### Receiving Events
-
-The `dora_node.events` field is a stream of incoming events.
-To wait for the next incoming event, call `dora_node.events->next()`:
-
-```c++
-auto event = dora_node.events->next();
-```
-
-The `next` function returns an opaque `DoraEvent` type, which cannot be inspected from C++ directly.
-Instead, use the following functions to read and destructure the event:
-
-- `event_type(event)` returns a `DoraEventType`, which describes the kind of event. For example, an event could be an input or a stop instruction.
-  - when receiving a `DoraEventType::AllInputsClosed`, the node should exit and not call `next` anymore
-- Events of type `DoraEventType::Input` can be downcasted using `event_as_input`:
-  ```c++
-  auto input = event_as_input(std::move(event));
-  ```
-  The function returns a `DoraInput` instance, which has an `id` and `data` field.
-  - The input `id` can be converted to a C++ string through `std::string(input.id)`.
-  - The `data` of inputs is currently of type [`rust::Vec<uint8_t>`](https://cxx.rs/binding/vec.html). Use the provided methods for reading or converting the data.
-    - **Note:** In the future, we plan to change the data type to the [Apache Arrow](https://arrow.apache.org/) data format to support typed inputs.
-
-### Sending Outputs
-
-Nodes can send outputs using the `send_output` output function and the `dora_node.send_output` field.
-Note that all outputs need to be listed in the dataflow YAML declaration file, otherwise an error will occur.
-
-**Example:**
-
-```c++
-// the data you want to send (NOTE: only byte vectors are supported right now)
-std::vector<uint8_t> out_vec{42};
-// create a Rust slice from the output vector
-rust::Slice<const uint8_t> out_slice{out_vec.data(), out_vec.size()};
-// send the slice as output
-auto result = send_output(dora_node.send_output, "output_id", out_slice);
-
-// check for errors
-auto error = std::string(result.error);
-if (!error.empty())
-{
-    std::cerr << "Error: " << error << std::endl;
-    return -1;
-}
-```
-
-## Using the ROS2 Bridge
-
-The `dora-ros2-bindings.h` contains function and struct definitions that allow interacting with ROS2 nodes.
-Currently, the bridge supports publishing and subscribing to ROS2 topics.
-In the future, we plan to support ROS2 services and ROS2 actions as well.
-
-### Initializing the ROS2 Context
-
-The first step is to initialize a ROS2 context:
-
-```c++
-auto ros2_context = init_ros2_context();
-```
-
-### Creating Nodes
-
-After initializing a ROS2 context, you can use it to create ROS2 nodes:
-
-```c++
-auto node = ros2_context->new_node("/ros2_demo", "turtle_teleop");
-```
-
-The first argument is the namespace of the node and the second argument is its name.
-
-### Creating Topics
-
-After creating a node, you can use one of the `create_topic_<TYPE>` functions to create a topic on it.
-The `<TYPE>` describes the message type that will be sent on the topic.
-The Dora ROS2 bridge automatically creates `create_topic_<TYPE>` functions for all messages types found in the sourced ROS2 environment.
-
-```c++
-auto vel_topic = node->create_topic_geometry_msgs_Twist("/turtle1", "cmd_vel", qos_default());
-```
-
-The first argument is the namespace of the topic and the second argument is its name.
-The third argument is the QoS (quality of service) setting for the topic.
-It can be adjusted as desired, for example:
-
-```c++
-auto qos = qos_default();
-qos.durability = Ros2Durability::Volatile;
-qos.liveliness = Ros2Liveliness::Automatic;
-auto vel_topic = node->create_topic_geometry_msgs_Twist("/turtle1", "cmd_vel", qos);
-```
-
-### Publish
-
-After creating a topic, it is possible to publish messages on it.
-First, create a publisher:
-
-```c++
-auto vel_publisher = node->create_publisher(vel_topic, qos);
-```
-
-The returned publisher is typed by the chosen topic.
-It will only accept messages of the topic's type, otherwise a compile error will occur.
-
-After creating a publisher, you can use the `publish` function to publish one or more messages.
-For example:
-
-```c++
-geometry_msgs::Twist twist = {
-    .linear = {.x = 1, .y = 0, .z = 0},
-    .angular = {.x = 0, .y = 0, .z = 0.5}
-};
-vel_publisher->publish(twist);
-```
-
-The `geometry_msgs::Twist` struct is automatically generated from the sourced ROS2 environment.
-Since the publisher is typed, its `publish` method only accepts `geometry_msgs::Twist` messages.
-
-
-### Subscriptions
-
-Subscribing to a topic is possible through the `create_subscription` function on nodes:
-
-```c++
-auto pose_topic = node->create_topic_turtlesim_Pose("/turtle1", "pose", qos_default());
-auto pose_subscription = node->create_subscription(pose_topic, qos_default(), event_stream);
-```
-
-The `topic` is the topic you want to subscribe to, created using a `create_topic_<TYPE>` function.
-The second argument is the quality of service setting, which can be customized as described above.
-
-The third parameter is the event stream that the received messages should be merged into.
-Multiple subscriptions can be merged into the same event stream.
-
-#### Combined Event Streams
-
-Combined event streams enable the merging of multiple event streams into one.
-The combined stream will then deliver messages from all sources, in order of arrival.
-
-You can create such a event stream from Dora's event stream using the `dora_events_into_combined` function:
-
-```c++
-auto event_stream = dora_events_into_combined(std::move(dora_node.events));
-```
-
-Alternatively, if you don't want to use Dora, you can also create an empty event stream:
-
-```c++
-auto event_stream = empty_combined_events();
-```
-
-**Note:** You should only use `empty_combined_events` if you're running your executable independent of Dora.
-Ignoring the events from the `dora_node.events` channel can result in unintended behavior.
-
-#### Receiving Messages from Combined Event Stream
-
-The merged event stream will receive all incoming events of the node, including Dora events and ROS2 messages.
-To wait for the next incoming event, use its `next` method:
-
-```c++
-auto event = event_stream.next();
-```
-
-This returns a `event` instance of type `CombinedEvent`, which can be downcasted to Dora events or ROS2 messages.
-To handle an event, you should check it's type and then downcast it:
-
-- To check for a Dora event, you can use the `is_dora()` function. If it returns `true`, you can downcast the combined event to a Dora event using the `downcast_dora` function.
-- ROS2 subscriptions support a `matches` function to check whether a combined event is an instance of the respective ROS2 subscription. If it returns true, you can downcast the event to the respective ROS2 message struct using the subscription's `downcast` function.
-
-**Example:**
-
-```c++
-if (event.is_dora())
-{
-    auto dora_event = downcast_dora(std::move(event));
-    // handle dora_event as described above
-    auto ty = event_type(dora_event);
-    if (ty == DoraEventType::Input)
-    {
-        auto input = event_as_input(std::move(dora_event));
-        // etc
-    }
-    // .. else if
-}
-else if (pose_subscription->matches(event))
-{
-    auto pose = pose_subscription->downcast(std::move(event));
-    std::cout << "Received pose x:" << pose.x << ", y:" << pose.y << std::endl;
-}
-else
-{
-    std::cout << "received unexpected event" << std::endl;
-}
-```
-
-### Constants
-
-Some ROS2 message definitions define constants, e.g. to specify the values of an enum-like integer field.
-The Dora ROS2 bridge exposes these constants in the generated bindings as functions.
-
-For example, the `STATUS_NO_FIX` constant of the [`NavSatStatus` message](https://docs.ros.org/en/jade/api/sensor_msgs/html/msg/NavSatStatus.html) can be accessed as follows:
-
-```c++
-assert((sensor_msgs::const_NavSatStatus_STATUS_NO_FIX() == -1));
-```
-
-(Note: Exposing them as C++ constants is not possible because it's [not supported by `cxx` yet](https://github.com/dtolnay/cxx/issues/1051).)
-
-### Service Clients
-
-To create a service client, use one of the `create_client_<TYPE>` functions.
-The `<TYPE>` describes the service type, which specifies the request and response types.
-The Dora ROS2 bridge automatically creates `create_client_<TYPE>` functions for all service types found in the sourced ROS2 environment.
-
-```c++
-auto add_two_ints = node->create_client_example_interfaces_AddTwoInts(
-  "/",
-  "add_two_ints",
-  qos,
-  merged_events
-);
-```
-
-- The first argument is the namespace of the service and the second argument is its name.
-- The third argument is the QoS (quality of service) setting for the service.
-  It can be set to `qos_default()` or adjusted as desired, for example:
-  ```c++
-  auto qos = qos_default();
-  qos.reliable = true;
-  qos.max_blocking_time = 0.1;
-  qos.keep_last = 1;
-  ```
-- The last argument is the [combined event stream](#combined-event-streams) that the received service responses should be merged into.
-
-#### Waiting for the Service
-
-In order to achieve reliable service communication, it is recommended to wait until the service is available before sending requests.
-Use the `wait_for_service()` method for that, e.g.:
-
-```c++
-add_two_ints->wait_for_service(node)
-```
-
-The given `node` must be the node on which the service was created.
-
-#### Sending Requests
-
-To send a request, use the `send_request` method:
-
-```c++
-add_two_ints->send_request(request);
-```
-
-The method sends the request asynchronously without waiting for a response.
-When the response is received, it is automatically sent to the [combined event stream](#combined-event-streams) that was given on client creation.
-
-#### Receiving Responses
-
-See the [_"Receiving Messages from Combined Event Stream"_](#receiving-messages-from-combined-event-stream) section for how to receive events from the combined event stream.
-To check if a received event is a service response, use the `matches` method.
-If it returns `true`, you can use the `downcast` method to convert the event to the correct service response type.
-
-Example:
-
-```c++
-if (add_two_ints->matches(event))
-{
-    auto response = add_two_ints->downcast(std::move(event));
-    std::cout << "Received sum response with value " << response.sum << std::endl;
-    ...
-}
-```

apis/c++/node/build.rs

@@ -1,167 +0,0 @@
-use std::path::{Path, PathBuf};
-
-fn main() {
-    let mut bridge_files = vec![PathBuf::from("src/lib.rs")];
-    #[cfg(feature = "ros2-bridge")]
-    bridge_files.push(ros2::generate());
-
-    let _build = cxx_build::bridges(&bridge_files);
-    println!("cargo:rerun-if-changed=src/lib.rs");
-
-    // rename header files
-    let src_dir = origin_dir();
-    let target_dir = src_dir.parent().unwrap();
-    std::fs::copy(src_dir.join("lib.rs.h"), target_dir.join("dora-node-api.h")).unwrap();
-    std::fs::copy(
-        src_dir.join("lib.rs.cc"),
-        target_dir.join("dora-node-api.cc"),
-    )
-    .unwrap();
-
-    #[cfg(feature = "ros2-bridge")]
-    ros2::generate_ros2_message_header(bridge_files.last().unwrap());
-
-    // to avoid unnecessary `mut` warning
-    bridge_files.clear();
-}
-
-fn origin_dir() -> PathBuf {
-    let default_target = std::env::var("CARGO_TARGET_DIR")
-        .map(PathBuf::from)
-        .unwrap_or_else(|_| {
-            let root = Path::new(env!("CARGO_MANIFEST_DIR"))
-                .ancestors()
-                .nth(3)
-                .unwrap();
-            root.join("target")
-        });
-    let cross_target = default_target
-        .join(std::env::var("TARGET").unwrap())
-        .join("cxxbridge")
-        .join("dora-node-api-cxx")
-        .join("src");
-
-    if cross_target.exists() {
-        cross_target
-    } else {
-        default_target
-            .join("cxxbridge")
-            .join("dora-node-api-cxx")
-            .join("src")
-    }
-}
-
-#[cfg(feature = "ros2-bridge")]
-mod ros2 {
-    use super::origin_dir;
-    use std::{
-        io::{BufRead, BufReader},
-        path::{Component, Path, PathBuf},
-    };
-
-    pub fn generate() -> PathBuf {
-        use rust_format::Formatter;
-        let paths = ament_prefix_paths();
-        let generated = dora_ros2_bridge_msg_gen::generate(paths.as_slice(), true);
-        let generated_string = rust_format::PrettyPlease::default()
-            .format_tokens(generated)
-            .unwrap();
-        let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
-        let target_file = out_dir.join("ros2_bindings.rs");
-        std::fs::write(&target_file, generated_string).unwrap();
-        println!(
-            "cargo:rustc-env=ROS2_BINDINGS_PATH={}",
-            target_file.display()
-        );
-
-        target_file
-    }
-
-    fn ament_prefix_paths() -> Vec<PathBuf> {
-        let ament_prefix_path: String = match std::env::var("AMENT_PREFIX_PATH") {
-            Ok(path) => path,
-            Err(std::env::VarError::NotPresent) => {
-                println!("cargo:warning='AMENT_PREFIX_PATH not set'");
-                String::new()
-            }
-            Err(std::env::VarError::NotUnicode(s)) => {
-                panic!(
-                    "AMENT_PREFIX_PATH is not valid unicode: `{}`",
-                    s.to_string_lossy()
-                );
-            }
-        };
-        println!("cargo:rerun-if-env-changed=AMENT_PREFIX_PATH");
-
-        let paths: Vec<_> = ament_prefix_path.split(':').map(PathBuf::from).collect();
-        for path in &paths {
-            println!("cargo:rerun-if-changed={}", path.display());
-        }
-
-        paths
-    }
-
-    pub fn generate_ros2_message_header(source_file: &Path) {
-        use std::io::Write as _;
-
-        let out_dir = source_file.parent().unwrap();
-        let relative_path = local_relative_path(&source_file)
-            .ancestors()
-            .nth(2)
-            .unwrap()
-            .join("out");
-        let header_path = out_dir
-            .join("cxxbridge")
-            .join("include")
-            .join("dora-node-api-cxx")
-            .join(&relative_path)
-            .join("ros2_bindings.rs.h");
-        let code_path = out_dir
-            .join("cxxbridge")
-            .join("sources")
-            .join("dora-node-api-cxx")
-            .join(&relative_path)
-            .join("ros2_bindings.rs.cc");
-
-        // copy message files to target directory
-        let target_path = origin_dir().parent().unwrap().join("dora-ros2-bindings.h");
-
-        std::fs::copy(&header_path, &target_path).unwrap();
-        println!("cargo:rerun-if-changed={}", header_path.display());
-
-        let node_header =
-            std::fs::File::open(target_path.with_file_name("dora-node-api.h")).unwrap();
-        let mut code_file = std::fs::File::open(&code_path).unwrap();
-        println!("cargo:rerun-if-changed={}", code_path.display());
-        let mut code_target_file =
-            std::fs::File::create(target_path.with_file_name("dora-ros2-bindings.cc")).unwrap();
-
-        // copy both the node header and the code file to prevent import errors
-        let mut header_reader = {
-            let mut reader = BufReader::new(node_header);
-
-            // read first line to skip `#pragma once`, which is not allowed in main files
-            let mut first_line = String::new();
-            reader.read_line(&mut first_line).unwrap();
-            assert_eq!(first_line.trim(), "#pragma once");
-
-            reader
-        };
-        std::io::copy(&mut header_reader, &mut code_target_file).unwrap();
-        std::io::copy(&mut code_file, &mut code_target_file).unwrap();
-        code_target_file.flush().unwrap();
-    }
-
-    // copy from cxx-build source
-    fn local_relative_path(path: &Path) -> PathBuf {
-        let mut rel_path = PathBuf::new();
-        for component in path.components() {
-            match component {
-                Component::Prefix(_) | Component::RootDir | Component::CurDir => {}
-                Component::ParentDir => drop(rel_path.pop()), // noop if empty
-                Component::Normal(name) => rel_path.push(name),
-            }
-        }
-        rel_path
-    }
-}

apis/c++/node/src/lib.rs

@@ -1,328 +0,0 @@
-use std::{any::Any, vec};
-
-use dora_node_api::{
-    self, Event, EventStream,
-    arrow::array::{AsArray, UInt8Array},
-    merged::{MergeExternal, MergedEvent},
-};
-use eyre::bail;
-
-#[cfg(feature = "ros2-bridge")]
-use dora_ros2_bridge::{_core, ros2_client};
-use futures_lite::{Stream, StreamExt, stream};
-
-#[cxx::bridge]
-#[allow(clippy::needless_lifetimes)]
-mod ffi {
-    struct DoraNode {
-        events: Box<Events>,
-        send_output: Box<OutputSender>,
-    }
-
-    pub enum DoraEventType {
-        Stop,
-        Input,
-        InputClosed,
-        Error,
-        Unknown,
-        AllInputsClosed,
-    }
-
-    struct DoraInput {
-        id: String,
-        data: Vec<u8>,
-    }
-
-    struct DoraResult {
-        error: String,
-    }
-
-    pub struct CombinedEvents {
-        events: Box<MergedEvents>,
-    }
-
-    pub struct CombinedEvent {
-        event: Box<MergedDoraEvent>,
-    }
-
-    extern "Rust" {
-        type Events;
-        type OutputSender;
-        type DoraEvent;
-        type MergedEvents;
-        type MergedDoraEvent;
-
-        fn init_dora_node() -> Result<DoraNode>;
-
-        fn dora_events_into_combined(events: Box<Events>) -> CombinedEvents;
-        fn empty_combined_events() -> CombinedEvents;
-        fn next(self: &mut Events) -> Box<DoraEvent>;
-        fn next_event(events: &mut Box<Events>) -> Box<DoraEvent>;
-        fn event_type(event: &Box<DoraEvent>) -> DoraEventType;
-        fn event_as_input(event: Box<DoraEvent>) -> Result<DoraInput>;
-        fn send_output(
-            output_sender: &mut Box<OutputSender>,
-            id: String,
-            data: &[u8],
-        ) -> DoraResult;
-
-        fn next(self: &mut CombinedEvents) -> CombinedEvent;
-
-        fn is_dora(self: &CombinedEvent) -> bool;
-        fn downcast_dora(event: CombinedEvent) -> Result<Box<DoraEvent>>;
-
-        unsafe fn send_arrow_output(
-            output_sender: &mut Box<OutputSender>,
-            id: String,
-            array_ptr: *mut u8,
-            schema_ptr: *mut u8,
-        ) -> DoraResult;
-
-        unsafe fn event_as_arrow_input(
-            event: Box<DoraEvent>,
-            out_array: *mut u8,
-            out_schema: *mut u8,
-        ) -> DoraResult;
-    }
-}
-
-#[cfg(feature = "ros2-bridge")]
-pub mod ros2 {
-    pub use dora_ros2_bridge::*;
-    include!(env!("ROS2_BINDINGS_PATH"));
-}
-
-fn init_dora_node() -> eyre::Result<ffi::DoraNode> {
-    let (node, events) = dora_node_api::DoraNode::init_from_env()?;
-    let events = Events(events);
-    let send_output = OutputSender(node);
-
-    Ok(ffi::DoraNode {
-        events: Box::new(events),
-        send_output: Box::new(send_output),
-    })
-}
-
-pub struct Events(EventStream);
-
-impl Events {
-    fn next(&mut self) -> Box<DoraEvent> {
-        Box::new(DoraEvent(self.0.recv()))
-    }
-}
-
-fn next_event(events: &mut Box<Events>) -> Box<DoraEvent> {
-    events.next()
-}
-
-fn dora_events_into_combined(events: Box<Events>) -> ffi::CombinedEvents {
-    let events = events.0.map(MergedEvent::Dora);
-    ffi::CombinedEvents {
-        events: Box::new(MergedEvents {
-            events: Some(Box::new(events)),
-            next_id: 1,
-        }),
-    }
-}
-
-fn empty_combined_events() -> ffi::CombinedEvents {
-    ffi::CombinedEvents {
-        events: Box::new(MergedEvents {
-            events: Some(Box::new(stream::empty())),
-            next_id: 1,
-        }),
-    }
-}
-
-pub struct DoraEvent(Option<Event>);
-
-fn event_type(event: &DoraEvent) -> ffi::DoraEventType {
-    match &event.0 {
-        Some(event) => match event {
-            Event::Stop(_) => ffi::DoraEventType::Stop,
-            Event::Input { .. } => ffi::DoraEventType::Input,
-            Event::InputClosed { .. } => ffi::DoraEventType::InputClosed,
-            Event::Error(_) => ffi::DoraEventType::Error,
-            _ => ffi::DoraEventType::Unknown,
-        },
-        None => ffi::DoraEventType::AllInputsClosed,
-    }
-}
-
-fn event_as_input(event: Box<DoraEvent>) -> eyre::Result<ffi::DoraInput> {
-    let Some(Event::Input { id, metadata, data }) = event.0 else {
-        bail!("not an input event");
-    };
-    let data = match metadata.type_info.data_type {
-        dora_node_api::arrow::datatypes::DataType::UInt8 => {
-            let array: &UInt8Array = data.as_primitive();
-            array.values().to_vec()
-        }
-        dora_node_api::arrow::datatypes::DataType::Null => {
-            vec![]
-        }
-        _ => {
-            todo!("dora C++ Node does not yet support higher level type of arrow. Only UInt8. 
-                The ultimate solution should be based on arrow FFI interface. Feel free to contribute :)")
-        }
-    };
-
-    Ok(ffi::DoraInput {
-        id: id.into(),
-        data,
-    })
-}
-
-unsafe fn event_as_arrow_input(
-    event: Box<DoraEvent>,
-    out_array: *mut u8,
-    out_schema: *mut u8,
-) -> ffi::DoraResult {
-    // Cast to Arrow FFI types
-    let out_array = out_array as *mut arrow::ffi::FFI_ArrowArray;
-    let out_schema = out_schema as *mut arrow::ffi::FFI_ArrowSchema;
-
-    let Some(Event::Input {
-        id: _,
-        metadata: _,
-        data,
-    }) = event.0
-    else {
-        return ffi::DoraResult {
-            error: "Not an input event".to_string(),
-        };
-    };
-
-    if out_array.is_null() || out_schema.is_null() {
-        return ffi::DoraResult {
-            error: "Received null output pointer".to_string(),
-        };
-    }
-
-    let array_data = data.to_data();
-
-    match arrow::ffi::to_ffi(&array_data.clone()) {
-        Ok((ffi_array, ffi_schema)) => {
-            std::ptr::write(out_array, ffi_array);
-            std::ptr::write(out_schema, ffi_schema);
-            ffi::DoraResult {
-                error: String::new(),
-            }
-        }
-        Err(e) => ffi::DoraResult {
-            error: format!("Error exporting Arrow array to C++: {e:?}"),
-        },
-    }
-}
-
-pub struct OutputSender(dora_node_api::DoraNode);
-
-fn send_output(sender: &mut Box<OutputSender>, id: String, data: &[u8]) -> ffi::DoraResult {
-    let result = sender
-        .0
-        .send_output_raw(id.into(), Default::default(), data.len(), |out| {
-            out.copy_from_slice(data)
-        });
-    let error = match result {
-        Ok(()) => String::new(),
-        Err(err) => format!("{err:?}"),
-    };
-    ffi::DoraResult { error }
-}
-
-pub struct MergedEvents {
-    events: Option<Box<dyn Stream<Item = MergedEvent<ExternalEvent>> + Unpin>>,
-    next_id: u32,
-}
-unsafe fn send_arrow_output(
-    sender: &mut Box<OutputSender>,
-    id: String,
-    array_ptr: *mut u8,
-    schema_ptr: *mut u8,
-) -> ffi::DoraResult {
-    let array_ptr = array_ptr as *mut arrow::ffi::FFI_ArrowArray;
-    let schema_ptr = schema_ptr as *mut arrow::ffi::FFI_ArrowSchema;
-
-    if array_ptr.is_null() || schema_ptr.is_null() {
-        return ffi::DoraResult {
-            error: "Received null Arrow array or schema pointer".to_string(),
-        };
-    }
-
-    let array = std::ptr::read(array_ptr);
-    let schema = std::ptr::read(schema_ptr);
-
-    std::ptr::write(array_ptr, std::mem::zeroed());
-    std::ptr::write(schema_ptr, std::mem::zeroed());
-
-    match arrow::ffi::from_ffi(array, &schema) {
-        Ok(array_data) => {
-            let arrow_array = arrow::array::make_array(array_data);
-            let result = sender
-                .0
-                .send_output(id.into(), Default::default(), arrow_array);
-            match result {
-                Ok(()) => ffi::DoraResult {
-                    error: String::new(),
-                },
-                Err(err) => ffi::DoraResult {
-                    error: format!("{err:?}"),
-                },
-            }
-        }
-        Err(e) => ffi::DoraResult {
-            error: format!("Error importing array from C++: {e:?}"),
-        },
-    }
-}
-
-impl MergedEvents {
-    fn next(&mut self) -> MergedDoraEvent {
-        let event = futures_lite::future::block_on(self.events.as_mut().unwrap().next());
-        MergedDoraEvent(event)
-    }
-
-    pub fn merge(&mut self, events: impl Stream<Item = Box<dyn Any>> + Unpin + 'static) -> u32 {
-        let id = self.next_id;
-        self.next_id += 1;
-        let events = Box::pin(events.map(move |event| ExternalEvent { event, id }));
-
-        let inner = self.events.take().unwrap();
-        let merged: Box<dyn Stream<Item = _> + Unpin + 'static> =
-            Box::new(inner.merge_external(events).map(|event| match event {
-                MergedEvent::Dora(event) => MergedEvent::Dora(event),
-                MergedEvent::External(event) => MergedEvent::External(event.flatten()),
-            }));
-        self.events = Some(merged);
-
-        id
-    }
-}
-
-impl ffi::CombinedEvents {
-    fn next(&mut self) -> ffi::CombinedEvent {
-        ffi::CombinedEvent {
-            event: Box::new(self.events.next()),
-        }
-    }
-}
-
-pub struct MergedDoraEvent(Option<MergedEvent<ExternalEvent>>);
-
-pub struct ExternalEvent {
-    pub event: Box<dyn Any>,
-    pub id: u32,
-}
-
-impl ffi::CombinedEvent {
-    fn is_dora(&self) -> bool {
-        matches!(&self.event.0, Some(MergedEvent::Dora(_)))
-    }
-}
-
-fn downcast_dora(event: ffi::CombinedEvent) -> eyre::Result<Box<DoraEvent>> {
-    match event.event.0 {
-        Some(MergedEvent::Dora(event)) => Ok(Box::new(DoraEvent(Some(event)))),
-        _ => eyre::bail!("not an external event"),
-    }
-}

apis/c++/operator/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "dora-operator-api-cxx"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-[lib]
-crate-type = ["staticlib"]
-
-[dependencies]
-cxx = "1.0.73"
-dora-operator-api = { workspace = true }
-
-[build-dependencies]
-cxx-build = "1.0.73"

apis/c++/operator/build.rs

@@ -1,4 +0,0 @@
-fn main() {
-    let _ = cxx_build::bridge("src/lib.rs");
-    println!("cargo:rerun-if-changed=src/lib.rs");
-}

apis/c++/operator/src/lib.rs

@@ -1,98 +0,0 @@
-#![cfg(not(test))]
-#![warn(unsafe_op_in_unsafe_fn)]
-
-use dora_operator_api::{
-    self, DoraOperator, DoraOutputSender, DoraStatus, Event, IntoArrow, register_operator,
-};
-use ffi::DoraSendOutputResult;
-
-#[cxx::bridge]
-#[allow(unsafe_op_in_unsafe_fn)]
-mod ffi {
-    struct DoraOnInputResult {
-        error: String,
-        stop: bool,
-    }
-
-    struct DoraSendOutputResult {
-        error: String,
-    }
-
-    extern "Rust" {
-        type OutputSender<'a, 'b>;
-
-        fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> DoraSendOutputResult;
-    }
-
-    unsafe extern "C++" {
-        include!("operator.h");
-
-        type Operator;
-
-        fn new_operator() -> UniquePtr<Operator>;
-
-        fn on_input(
-            op: Pin<&mut Operator>,
-            id: &str,
-            data: &[u8],
-            output_sender: &mut OutputSender,
-        ) -> DoraOnInputResult;
-    }
-}
-
-pub struct OutputSender<'a, 'b>(&'a mut DoraOutputSender<'b>);
-
-fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> DoraSendOutputResult {
-    let error = sender
-        .0
-        .send(id.into(), data.to_owned().into_arrow())
-        .err()
-        .unwrap_or_default();
-    DoraSendOutputResult { error }
-}
-
-register_operator!(OperatorWrapper);
-
-struct OperatorWrapper {
-    operator: cxx::UniquePtr<ffi::Operator>,
-}
-
-impl Default for OperatorWrapper {
-    fn default() -> Self {
-        Self {
-            operator: ffi::new_operator(),
-        }
-    }
-}
-
-impl DoraOperator for OperatorWrapper {
-    fn on_event(
-        &mut self,
-        event: &Event,
-        output_sender: &mut DoraOutputSender,
-    ) -> Result<DoraStatus, std::string::String> {
-        match event {
-            Event::Input { id, data } => {
-                let operator = self.operator.as_mut().unwrap();
-                let mut output_sender = OutputSender(output_sender);
-                let data: &[u8] = data
-                    .try_into()
-                    .map_err(|err| format!("expected byte array: {err}"))?;
-
-                let result = ffi::on_input(operator, id, data, &mut output_sender);
-                if result.error.is_empty() {
-                    Ok(match result.stop {
-                        false => DoraStatus::Continue,
-                        true => DoraStatus::Stop,
-                    })
-                } else {
-                    Err(result.error)
-                }
-            }
-            _ => {
-                // ignore other events for now
-                Ok(DoraStatus::Continue)
-            }
-        }
-    }
-}

apis/c/node/Cargo.toml

@@ -1,28 +0,0 @@
-[package]
-name = "dora-node-api-c"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-crate-type = ["staticlib", "lib"]
-
-
-[features]
-default = ["tracing"]
-tracing = ["dora-node-api/tracing"]
-
-[dependencies]
-eyre = "0.6.8"
-tracing = "0.1.33"
-arrow-array = { workspace = true }
-
-[dependencies.dora-node-api]
-workspace = true

apis/c/node/node_api.h

@@ -1,22 +0,0 @@
-#include <stddef.h>
-
-void *init_dora_context_from_env();
-void free_dora_context(void *dora_context);
-
-void *dora_next_event(void *dora_context);
-void free_dora_event(void *dora_event);
-
-enum DoraEventType
-{
-    DoraEventType_Stop,
-    DoraEventType_Input,
-    DoraEventType_InputClosed,
-    DoraEventType_Error,
-    DoraEventType_Unknown,
-};
-enum DoraEventType read_dora_event_type(void *dora_event);
-
-void read_dora_input_id(void *dora_event, char **out_ptr, size_t *out_len);
-void read_dora_input_data(void *dora_event, char **out_ptr, size_t *out_len);
-unsigned long long read_dora_input_timestamp(void *dora_event);
-int dora_send_output(void *dora_context, char *id_ptr, size_t id_len, char *data_ptr, size_t data_len);

apis/c/node/src/lib.rs

@@ -1,277 +0,0 @@
-#![deny(unsafe_op_in_unsafe_fn)]
-
-use arrow_array::UInt8Array;
-use dora_node_api::{DoraNode, Event, EventStream, arrow::array::AsArray};
-use eyre::Context;
-use std::{ffi::c_void, ptr, slice};
-
-pub const HEADER_NODE_API: &str = include_str!("../node_api.h");
-
-struct DoraContext {
-    node: &'static mut DoraNode,
-    events: EventStream,
-}
-
-/// Initializes a dora context from the environment variables that were set by
-/// the dora-coordinator.
-///
-/// Returns a pointer to the dora context on success. This pointer can be
-/// used to call dora API functions that expect a `context` argument. Any
-/// other use is prohibited. To free the dora context when it is no longer
-/// needed, use the [`free_dora_context`] function.
-///
-/// On error, a null pointer is returned.
-#[unsafe(no_mangle)]
-pub extern "C" fn init_dora_context_from_env() -> *mut c_void {
-    let context = || {
-        let (node, events) = DoraNode::init_from_env()?;
-        let node = Box::leak(Box::new(node));
-        Result::<_, eyre::Report>::Ok(DoraContext { node, events })
-    };
-    let context = match context().context("failed to initialize node") {
-        Ok(n) => n,
-        Err(err) => {
-            let err: eyre::Error = err;
-            tracing::error!("{err:?}");
-            return ptr::null_mut();
-        }
-    };
-
-    Box::into_raw(Box::new(context)).cast()
-}
-
-/// Frees the given dora context.
-///
-/// ## Safety
-///
-/// Only pointers created through [`init_dora_context_from_env`] are allowed
-/// as arguments. Each context pointer must be freed exactly once. After
-/// freeing, the pointer must not be used anymore.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn free_dora_context(context: *mut c_void) {
-    let context: Box<DoraContext> = unsafe { Box::from_raw(context.cast()) };
-    // drop all fields except for `node`
-    let DoraContext { node, .. } = *context;
-    // convert the `'static` reference back to a Box, then drop it
-    let _ = unsafe { Box::from_raw(node as *const DoraNode as *mut DoraNode) };
-}
-
-/// Waits for the next incoming event for the node.
-///
-/// Returns a pointer to the event on success. This pointer must not be used
-/// directly. Instead, use the `read_dora_event_*` functions to read out the
-/// type and payload of the event. When the event is not needed anymore, use
-/// [`free_dora_event`] to free it again.
-///
-/// Returns a null pointer when all event streams were closed. This means that
-/// no more event will be available. Nodes typically react by stopping.
-///
-/// ## Safety
-///
-/// The `context` argument must be a dora context created through
-/// [`init_dora_context_from_env`]. The context must be still valid, i.e., not
-/// freed yet.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn dora_next_event(context: *mut c_void) -> *mut c_void {
-    let context: &mut DoraContext = unsafe { &mut *context.cast() };
-    match context.events.recv() {
-        Some(event) => Box::into_raw(Box::new(event)).cast(),
-        None => ptr::null_mut(),
-    }
-}
-
-/// Reads out the type of the given event.
-///
-/// ## Safety
-///
-/// The `event` argument must be a dora event received through
-/// [`dora_next_event`]. The event must be still valid, i.e., not
-/// freed yet.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn read_dora_event_type(event: *const ()) -> EventType {
-    let event: &Event = unsafe { &*event.cast() };
-    match event {
-        Event::Stop(_) => EventType::Stop,
-        Event::Input { .. } => EventType::Input,
-        Event::InputClosed { .. } => EventType::InputClosed,
-        Event::Error(_) => EventType::Error,
-        _ => EventType::Unknown,
-    }
-}
-
-#[repr(C)]
-pub enum EventType {
-    Stop,
-    Input,
-    InputClosed,
-    Error,
-    Unknown,
-}
-
-/// Reads out the ID of the given input event.
-///
-/// Writes the `out_ptr` and `out_len` with the start pointer and length of the
-/// ID string of the input. The ID is guaranteed to be valid UTF-8.
-///
-/// Writes a null pointer and length `0` if the given event is not an input event.
-///
-/// ## Safety
-///
-/// - The `event` argument must be a dora event received through
-///   [`dora_next_event`]. The event must be still valid, i.e., not
-///   freed yet. The returned `out_ptr` must not be used after
-///   freeing the `event`, since it points directly into the event's
-///   memory.
-///
-/// - Note: `Out_ptr` is not a null-terminated string. The length of the string
-///   is given by `out_len`.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn read_dora_input_id(
-    event: *const (),
-    out_ptr: *mut *const u8,
-    out_len: *mut usize,
-) {
-    let event: &Event = unsafe { &*event.cast() };
-    match event {
-        Event::Input { id, .. } => {
-            let id = id.as_str().as_bytes();
-            let ptr = id.as_ptr();
-            let len = id.len();
-            unsafe {
-                *out_ptr = ptr;
-                *out_len = len;
-            }
-        }
-        _ => unsafe {
-            *out_ptr = ptr::null();
-            *out_len = 0;
-        },
-    }
-}
-
-/// Reads out the data of the given input event.
-///
-/// Writes the `out_ptr` and `out_len` with the start pointer and length of the
-/// input's data array. The data array is a raw byte array, whose format
-/// depends on the source operator/node.
-///
-/// Writes a null pointer and length `0` if the given event is not an input event
-/// or when an input event has no associated data.
-///
-/// ## Safety
-///
-/// The `event` argument must be a dora event received through
-/// [`dora_next_event`]. The event must be still valid, i.e., not
-/// freed yet. The returned `out_ptr` must not be used after
-/// freeing the `event`, since it points directly into the event's
-/// memory.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn read_dora_input_data(
-    event: *const (),
-    out_ptr: *mut *const u8,
-    out_len: *mut usize,
-) {
-    let event: &Event = unsafe { &*event.cast() };
-    match event {
-        Event::Input { data, metadata, .. } => match metadata.type_info.data_type {
-            dora_node_api::arrow::datatypes::DataType::UInt8 => {
-                let array: &UInt8Array = data.as_primitive();
-                let ptr = array.values().as_ptr();
-                unsafe {
-                    *out_ptr = ptr;
-                    *out_len = metadata.type_info.len;
-                }
-            }
-            dora_node_api::arrow::datatypes::DataType::Null => unsafe {
-                *out_ptr = ptr::null();
-                *out_len = 0;
-            },
-            _ => {
-                todo!("dora C++ Node does not yet support higher level type of arrow. Only UInt8. 
-                The ultimate solution should be based on arrow FFI interface. Feel free to contribute :)")
-            }
-        },
-        _ => unsafe {
-            *out_ptr = ptr::null();
-            *out_len = 0;
-        },
-    }
-}
-
-/// Reads out the timestamp of the given input event from metadata.
-///
-/// ## Safety
-///
-/// Return `0` if the given event is not an input event.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn read_dora_input_timestamp(event: *const ()) -> core::ffi::c_ulonglong {
-    let event: &Event = unsafe { &*event.cast() };
-    match event {
-        Event::Input { metadata, .. } => metadata.timestamp().get_time().as_u64(),
-        _ => 0,
-    }
-}
-
-/// Frees the given dora event.
-///
-/// ## Safety
-///
-/// Only pointers created through [`dora_next_event`] are allowed
-/// as arguments. Each context pointer must be freed exactly once. After
-/// freeing, the pointer and all derived pointers must not be used anymore.
-/// This also applies to the `read_dora_event_*` functions, which return
-/// pointers into the original event structure.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn free_dora_event(event: *mut c_void) {
-    let _: Box<Event> = unsafe { Box::from_raw(event.cast()) };
-}
-
-/// Sends the given output to subscribed dora nodes/operators.
-///
-/// The `id_ptr` and `id_len` fields must be the start pointer and length of an
-/// UTF8-encoded string. The ID string must correspond to one of the node's
-/// outputs specified in the dataflow YAML file.
-///
-/// The `data_ptr` and `data_len` fields must be the start pointer and length
-/// a byte array. The dora API sends this data as-is, without any processing.
-///
-/// ## Safety
-///
-/// - The `id_ptr` and `id_len` fields must be the start pointer and length of an
-///   UTF8-encoded string.
-/// - The `data_ptr` and `data_len` fields must be the start pointer and length
-///   a byte array.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn dora_send_output(
-    context: *mut c_void,
-    id_ptr: *const u8,
-    id_len: usize,
-    data_ptr: *const u8,
-    data_len: usize,
-) -> isize {
-    match unsafe { try_send_output(context, id_ptr, id_len, data_ptr, data_len) } {
-        Ok(()) => 0,
-        Err(err) => {
-            tracing::error!("{err:?}");
-            -1
-        }
-    }
-}
-
-unsafe fn try_send_output(
-    context: *mut c_void,
-    id_ptr: *const u8,
-    id_len: usize,
-    data_ptr: *const u8,
-    data_len: usize,
-) -> eyre::Result<()> {
-    let context: &mut DoraContext = unsafe { &mut *context.cast() };
-    let id = std::str::from_utf8(unsafe { slice::from_raw_parts(id_ptr, id_len) })?;
-    let output_id = id.to_owned().into();
-    let data = unsafe { slice::from_raw_parts(data_ptr, data_len) };
-    context
-        .node
-        .send_output_raw(output_id, Default::default(), data.len(), |out| {
-            out.copy_from_slice(data);
-        })
-}

apis/c/operator/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "dora-operator-api-c"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-description = "C API implementation for Dora Operator"
-documentation.workspace = true
-license.workspace = true
-repository.workspace = true
-
-[lib]
-crate-type = ["staticlib", "lib"]
-
-[dependencies]
-dora-operator-api-types = { workspace = true }
-
-[build-dependencies]
-dora-operator-api-types = { workspace = true }

apis/c/operator/build.rs

@@ -1,11 +0,0 @@
-use std::path::Path;
-
-fn main() {
-    dora_operator_api_types::generate_headers(Path::new("operator_types.h"))
-        .expect("failed to create operator_types.h");
-
-    // don't rebuild on changes (otherwise we rebuild on every run as we're
-    // writing the `operator_types.h` file; cargo will still rerun this script
-    // when the `dora_operator_api_types` crate changes)
-    println!("cargo:rerun-if-changed=build.rs");
-}

apis/c/operator/operator_api.h

@@ -1,36 +0,0 @@
-#ifndef __RUST_DORA_OPERATOR_API_C_WRAPPER__
-#define __RUST_DORA_OPERATOR_API_C_WRAPPER__
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-#include <stddef.h>
-#include "operator_types.h"
-
-#ifdef _WIN32
-#define EXPORT __declspec(dllexport)
-#else
-#define EXPORT __attribute__((visibility("default")))
-#endif
-
-    EXPORT DoraInitResult_t dora_init_operator(void);
-
-    EXPORT DoraResult_t dora_drop_operator(void *operator_context);
-
-    EXPORT OnEventResult_t dora_on_event(
-        RawEvent_t *event,
-        const SendOutput_t *send_output,
-        void *operator_context);
-
-    static void __dora_type_assertions()
-    {
-        DoraInitOperator_t __dora_init_operator = {.init_operator = dora_init_operator};
-        DoraDropOperator_t __dora_drop_operator = {.drop_operator = dora_drop_operator};
-        DoraOnEvent_t __dora_on_event = {.on_event = dora_on_event};
-    }
-#ifdef __cplusplus
-} /* extern \"C\" */
-#endif
-
-#endif /* __RUST_DORA_OPERATOR_API_C_WRAPPER__ */

apis/c/operator/operator_types.h

@@ -1,180 +0,0 @@
-/*! \file */
-/*******************************************
- *                                         *
- *  File auto-generated by `::safer_ffi`.  *
- *                                         *
- *  Do not manually edit this file.        *
- *                                         *
- *******************************************/
-
-#ifndef __RUST_DORA_OPERATOR_API_C__
-#define __RUST_DORA_OPERATOR_API_C__
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-#include <stddef.h>
-#include <stdint.h>
-
-/** \brief
- *  Same as [`Vec<T>`][`rust::Vec`], but with guaranteed `#[repr(C)]` layout
- */
-typedef struct Vec_uint8 {
-    /** <No documentation available> */
-    uint8_t * ptr;
-
-    /** <No documentation available> */
-    size_t len;
-
-    /** <No documentation available> */
-    size_t cap;
-} Vec_uint8_t;
-
-/** <No documentation available> */
-typedef struct DoraResult {
-    /** <No documentation available> */
-    Vec_uint8_t * error;
-} DoraResult_t;
-
-/** <No documentation available> */
-typedef struct DoraDropOperator {
-    /** <No documentation available> */
-    DoraResult_t (*drop_operator)(void *);
-} DoraDropOperator_t;
-
-/** <No documentation available> */
-typedef struct DoraInitResult {
-    /** <No documentation available> */
-    DoraResult_t result;
-
-    /** <No documentation available> */
-    void * operator_context;
-} DoraInitResult_t;
-
-/** <No documentation available> */
-typedef struct DoraInitOperator {
-    /** <No documentation available> */
-    DoraInitResult_t (*init_operator)(void);
-} DoraInitOperator_t;
-
-/** <No documentation available> */
-/** \remark Has the same ABI as `uint8_t` **/
-#ifdef DOXYGEN
-typedef
-#endif
-enum DoraStatus {
-    /** <No documentation available> */
-    DORA_STATUS_CONTINUE = 0,
-    /** <No documentation available> */
-    DORA_STATUS_STOP = 1,
-    /** <No documentation available> */
-    DORA_STATUS_STOP_ALL = 2,
-}
-#ifndef DOXYGEN
-; typedef uint8_t
-#endif
-DoraStatus_t;
-
-/** <No documentation available> */
-typedef struct OnEventResult {
-    /** <No documentation available> */
-    DoraResult_t result;
-
-    /** <No documentation available> */
-    DoraStatus_t status;
-} OnEventResult_t;
-
-/** <No documentation available> */
-typedef struct Input Input_t;
-
-
-#include <stdbool.h>
-
-/** <No documentation available> */
-typedef struct RawEvent {
-    /** <No documentation available> */
-    Input_t * input;
-
-    /** <No documentation available> */
-    Vec_uint8_t input_closed;
-
-    /** <No documentation available> */
-    bool stop;
-
-    /** <No documentation available> */
-    Vec_uint8_t error;
-} RawEvent_t;
-
-/** <No documentation available> */
-typedef struct Output Output_t;
-
-/** \brief
- *  `Arc<dyn Send + Sync + Fn(A1) -> Ret>`
- */
-typedef struct ArcDynFn1_DoraResult_Output {
-    /** <No documentation available> */
-    void * env_ptr;
-
-    /** <No documentation available> */
-    DoraResult_t (*call)(void *, Output_t);
-
-    /** <No documentation available> */
-    void (*release)(void *);
-
-    /** <No documentation available> */
-    void (*retain)(void *);
-} ArcDynFn1_DoraResult_Output_t;
-
-/** <No documentation available> */
-typedef struct SendOutput {
-    /** <No documentation available> */
-    ArcDynFn1_DoraResult_Output_t send_output;
-} SendOutput_t;
-
-/** <No documentation available> */
-typedef struct DoraOnEvent {
-    /** <No documentation available> */
-    OnEventResult_t (*on_event)(RawEvent_t *, SendOutput_t const *, void *);
-} DoraOnEvent_t;
-
-/** <No documentation available> */
-typedef struct Metadata {
-    /** <No documentation available> */
-    Vec_uint8_t open_telemetry_context;
-} Metadata_t;
-
-/** <No documentation available> */
-void
-dora_free_data (
-    Vec_uint8_t _data);
-
-/** <No documentation available> */
-void
-dora_free_input_id (
-    char * _input_id);
-
-/** <No documentation available> */
-Vec_uint8_t
-dora_read_data (
-    Input_t * input);
-
-/** <No documentation available> */
-char *
-dora_read_input_id (
-    Input_t const * input);
-
-/** <No documentation available> */
-DoraResult_t
-dora_send_operator_output (
-    SendOutput_t const * send_output,
-    char const * id,
-    uint8_t const * data_ptr,
-    size_t data_len);
-
-
-#ifdef __cplusplus
-} /* extern \"C\" */
-#endif
-
-#endif /* __RUST_DORA_OPERATOR_API_C__ */

apis/c/operator/src/lib.rs

@@ -1,4 +0,0 @@
-pub const HEADER_OPERATOR_API: &str = include_str!("../operator_api.h");
-pub const HEADER_OPERATOR_TYPES: &str = include_str!("../operator_types.h");
-
-pub use dora_operator_api_types;

apis/python/node/Cargo.toml

@@ -1,42 +0,0 @@
-[package]
-version.workspace = true
-name = "dora-node-api-python"
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[features]
-default = ["tracing", "metrics", "telemetry", "async"]
-tracing = ["dora-node-api/tracing"]
-metrics = ["dora-node-api/metrics"]
-async = ["pyo3/experimental-async"]
-telemetry = ["dora-runtime/telemetry"]
-
-[dependencies]
-dora-node-api = { workspace = true }
-dora-operator-api-python = { workspace = true }
-pyo3.workspace = true
-eyre = "0.6"
-serde_yaml = { workspace = true }
-flume = "0.10.14"
-dora-runtime = { workspace = true, features = ["tracing", "metrics", "python"] }
-dora-cli = { workspace = true }
-dora-download = { workspace = true }
-arrow = { workspace = true, features = ["pyarrow"] }
-pythonize = { workspace = true }
-futures = "0.3.28"
-dora-ros2-bridge-python = { workspace = true }
-pyo3_special_method_derive = "0.4.3"
-tokio = { version = "1.24.2", features = ["rt"] }
-
-[build-dependencies]
-pyo3-build-config = "0.23"
-
-[lib]
-name = "dora"
-crate-type = ["cdylib"]

apis/python/node/README.md

@@ -1,19 +0,0 @@
-This crate corresponds to the Node API for Dora.
-
-## Building
-
-To build the Python module for development:
-
-```bash
-uv venv --seed -p 3.11
-uv pip install -e .
-```
-
-## Type hinting
-
-Type hinting requires to run a second step
-
-```bash
-python generate_stubs.py dora dora/__init__.pyi
-maturin develop
-```

apis/python/node/build.rs

@@ -1,3 +0,0 @@
-fn main() {
-    pyo3_build_config::add_extension_module_link_args();
-}

apis/python/node/dora/__init__.py

@@ -1,42 +0,0 @@
-""" 
-# dora-rs.
-
-This is the dora python client for interacting with dora dataflow.
-You can install it via:
-```bash
-pip install dora-rs
-```.
-"""
-
-from enum import Enum
-
-from .dora import *
-from .dora import (
-    Node,
-    Ros2Context,
-    Ros2Durability,
-    Ros2Liveliness,
-    Ros2Node,
-    Ros2NodeOptions,
-    Ros2Publisher,
-    Ros2QosPolicies,
-    Ros2Subscription,
-    Ros2Topic,
-    __author__,
-    __version__,
-    start_runtime,
-)
-
-
-class DoraStatus(Enum):
-    """Dora status to indicate if operator `on_input` loop should be stopped.
-
-    Args:
-        Enum (u8): Status signaling to dora operator to
-        stop or continue the operator.
-
-    """
-
-    CONTINUE = 0
-    STOP = 1
-    STOP_ALL = 2

apis/python/node/dora/__init__.pyi

@@ -1,335 +0,0 @@
-import typing
-
-import pyarrow
-
-import dora
-
-@typing.final
-class Enum:
-    """Generic enumeration.
-
-    Derive from this class to define new enumerations.
-    """
-
-    __members__: mappingproxy = ...
-
-@typing.final
-class Node:
-    """The custom node API lets you integrate `dora` into your application.
-    
-    It allows you to retrieve input and send output in any fashion you want.
-
-    Use with:
-
-    ```python
-    from dora import Node
-
-    node = Node()
-    ```
-    """
-
-    def __init__(self, node_id: str=None) -> None:
-        """Use the custom node API to embed `dora` into your application.
-
-        It allows you to retrieve input and send output in any fashion you want.
-
-        Use with:
-
-        ```python
-        from dora import Node
-
-        node = Node()
-        ```
-        """
-
-    def dataflow_descriptor(self) -> dict:
-        """Return the full dataflow descriptor that this node is part of.
-
-        This method returns the parsed dataflow YAML file.
-        """
-
-    def dataflow_id(self) -> str:
-        """Return the dataflow id."""
-
-    def merge_external_events(self, subscription: dora.Ros2Subscription) -> None:
-        """Merge an external event stream with dora main loop.
-
-        This currently only work with ROS2.
-        """
-
-    def next(self, timeout: float=None) -> dict:
-        """`.next()` gives you the next input that the node has received.
-
-        It blocks until the next event becomes available.
-        You can use timeout in seconds to return if no input is available.
-        It will return `None` when all senders has been dropped.
-
-        ```python
-        event = node.next()
-        ```
-
-        You can also iterate over the event stream with a loop
-
-        ```python
-        for event in node:
-        match event["type"]:
-        case "INPUT":
-        match event["id"]:
-        case "image":
-        ```
-        """
-
-    def send_output(self, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None:
-        """`send_output` send data from the node.
-
-        ```python
-        Args:
-        output_id: str,
-        data: pyarrow.Array,
-        metadata: Option[Dict],
-        ```
-
-        ex:
-
-        ```python
-        node.send_output("string", b"string", {"open_telemetry_context": "7632e76"})
-        ```
-        """
-
-    def __iter__(self) -> typing.Any:
-        """Implement iter(self)."""
-
-    def __next__(self) -> typing.Any:
-        """Implement next(self)."""
-
-@typing.final
-class Ros2Context:
-    """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
-
-    By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
-
-    AMENT_PREFIX_PATH folder structure should be the following:
-
-    - For messages: <namespace>/msg/<name>.msg
-    - For services: <namespace>/srv/<name>.srv
-
-    You can also use `ros_paths` if you don't want to use env variable.
-
-    warning::
-    dora Ros2 bridge functionality is considered **unstable**. It may be changed
-    at any point without it being considered a breaking change.
-
-    ```python
-    context = Ros2Context()
-    ```
-    """
-
-    def __init__(self, ros_paths: list[str]=None) -> None:
-        """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
-
-        By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
-
-        AMENT_PREFIX_PATH folder structure should be the following:
-
-        - For messages: <namespace>/msg/<name>.msg
-        - For services: <namespace>/srv/<name>.srv
-
-        You can also use `ros_paths` if you don't want to use env variable.
-
-        warning::
-        dora Ros2 bridge functionality is considered **unstable**. It may be changed
-        at any point without it being considered a breaking change.
-
-        ```python
-        context = Ros2Context()
-        ```
-        """
-
-    def new_node(self, name: str, namespace: str, options: dora.Ros2NodeOptions) -> dora.Ros2Node:
-        """Create a new ROS2 node.
-
-        ```python
-        ros2_node = ros2_context.new_node(
-        "turtle_teleop",
-        "/ros2_demo",
-        Ros2NodeOptions(rosout=True),
-        )
-        ```
-
-        warning::
-        dora Ros2 bridge functionality is considered **unstable**. It may be changed
-        at any point without it being considered a breaking change.
-        """
-
-@typing.final
-class Ros2Durability:
-    """DDS 2.2.3.4 DURABILITY."""
-
-    def __eq__(self, value: object) -> bool:
-        """Return self==value."""
-
-    def __ge__(self, value: typing.Any) -> bool:
-        """Return self>=value."""
-
-    def __gt__(self, value: typing.Any) -> bool:
-        """Return self>value."""
-
-    def __int__(self) -> None:
-        """int(self)."""
-
-    def __le__(self, value: typing.Any) -> bool:
-        """Return self<=value."""
-
-    def __lt__(self, value: typing.Any) -> bool:
-        """Return self<value."""
-
-    def __ne__(self, value: object) -> bool:
-        """Return self!=value."""
-
-    Persistent: Ros2Durability = ...
-    Transient: Ros2Durability = ...
-    TransientLocal: Ros2Durability = ...
-    Volatile: Ros2Durability = ...
-
-@typing.final
-class Ros2Liveliness:
-    """DDS 2.2.3.11 LIVELINESS."""
-
-    def __eq__(self, value: object) -> bool:
-        """Return self==value."""
-
-    def __ge__(self, value: typing.Any) -> bool:
-        """Return self>=value."""
-
-    def __gt__(self, value: typing.Any) -> bool:
-        """Return self>value."""
-
-    def __int__(self) -> None:
-        """int(self)."""
-
-    def __le__(self, value: typing.Any) -> bool:
-        """Return self<=value."""
-
-    def __lt__(self, value: typing.Any) -> bool:
-        """Return self<value."""
-
-    def __ne__(self, value: object) -> bool:
-        """Return self!=value."""
-
-    Automatic: Ros2Liveliness = ...
-    ManualByParticipant: Ros2Liveliness = ...
-    ManualByTopic: Ros2Liveliness = ...
-
-@typing.final
-class Ros2Node:
-    """ROS2 Node.
-
-    warnings::
-    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-    at any point without it being considered a breaking change.
-    - There's a known issue about ROS2 nodes not being discoverable by ROS2
-    See: https://github.com/jhelovuo/ros2-client/issues/4
-    """
-
-    def create_publisher(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Publisher:
-        """Create a ROS2 publisher.
-
-        ```python
-        pose_publisher = ros2_node.create_publisher(turtle_pose_topic)
-        ```
-        warnings:
-        - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-        at any point without it being considered a breaking change.
-        """
-
-    def create_subscription(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription:
-        """Create a ROS2 subscription.
-
-        ```python
-        pose_reader = ros2_node.create_subscription(turtle_pose_topic)
-        ```
-
-        Warnings:
-        - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-        at any point without it being considered a breaking change.
-
-        """
-
-    def create_topic(self, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic:
-        """Create a ROS2 topic to connect to.
-
-        ```python
-        turtle_twist_topic = ros2_node.create_topic(
-        "/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos
-        )
-        ```
-        """
-
-@typing.final
-class Ros2NodeOptions:
-    """ROS2 Node Options."""
-
-    def __init__(self, rosout: bool=None) -> None:
-        """ROS2 Node Options."""
-
-@typing.final
-class Ros2Publisher:
-    """ROS2 Publisher.
-
-    Warnings:
-    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-    at any point without it being considered a breaking change.
-
-    """
-
-    def publish(self, data: pyarrow.Array) -> None:
-        """Publish a message into ROS2 topic.
-
-        Remember that the data format should respect the structure of the ROS2 message using an arrow Structure.
-
-        ex:
-        ```python
-        gripper_command.publish(
-        pa.array(
-        [
-        {
-        "name": "gripper",
-        "cmd": np.float32(5),
-        }
-        ]
-        ),
-        )
-        ```
-        """
-
-@typing.final
-class Ros2QosPolicies:
-    """ROS2 QoS Policy."""
-
-    def __init__(self, durability: dora.Ros2Durability=None, liveliness: dora.Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> dora.Ros2QoSPolicies:
-        """ROS2 QoS Policy."""
-
-@typing.final
-class Ros2Subscription:
-    """ROS2 Subscription.
-
-    Warnings:
-    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-    at any point without it being considered a breaking change.
-
-    """
-
-    def next(self):...
-
-@typing.final
-class Ros2Topic:
-    """ROS2 Topic.
-
-    Warnings:
-    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
-    at any point without it being considered a breaking change.
-
-    """
-
-def start_runtime() -> None:
-    """Start a runtime for Operators."""

apis/python/node/dora/cuda.py

@@ -1,94 +0,0 @@
-"""TODO: Add docstring."""
-
-import pyarrow as pa
-
-# Make sure to install torch with cuda
-import torch
-from numba.cuda import to_device
-
-# Make sure to install numba with cuda
-from numba.cuda.cudadrv.devicearray import DeviceNDArray
-from numba.cuda.cudadrv.devices import get_context
-from numba.cuda.cudadrv.driver import IpcHandle
-
-
-import json
-
-from contextlib import contextmanager
-from typing import ContextManager
-
-
-def torch_to_ipc_buffer(tensor: torch.TensorType) -> tuple[pa.array, dict]:
-    """Convert a Pytorch tensor into a pyarrow buffer containing the IPC handle
-    and its metadata.
-
-    Example Use:
-    ```python
-    torch_tensor = torch.tensor(random_data, dtype=torch.int64, device="cuda")
-    ipc_buffer, metadata = torch_to_ipc_buffer(torch_tensor)
-    node.send_output("latency", ipc_buffer, metadata)
-    ```
-    """
-    device_arr = to_device(tensor)
-    ipch = get_context().get_ipc_handle(device_arr.gpu_data)
-    _, handle, size, source_info, offset = ipch.__reduce__()[1]
-    metadata = {
-        "shape": device_arr.shape,
-        "strides": device_arr.strides,
-        "dtype": device_arr.dtype.str,
-        "size": size,
-        "offset": offset,
-        "source_info": json.dumps(source_info),
-    }
-    return pa.array(handle, pa.int8()), metadata
-
-
-def ipc_buffer_to_ipc_handle(handle_buffer: pa.array, metadata: dict) -> IpcHandle:
-    """Convert a buffer containing a serialized handler into cuda IPC Handle.
-
-    example use:
-    ```python
-    from dora.cuda import ipc_buffer_to_ipc_handle, open_ipc_handle
-
-    event = node.next()
-
-    ipc_handle = ipc_buffer_to_ipc_handle(event["value"], event["metadata"])
-    with open_ipc_handle(ipc_handle, event["metadata"]) as tensor:
-        pass
-    ```
-    """
-    handle = handle_buffer.to_pylist()
-    return IpcHandle._rebuild(
-        handle,
-        metadata["size"],
-        json.loads(metadata["source_info"]),
-        metadata["offset"],
-    )
-
-
-@contextmanager
-def open_ipc_handle(
-    ipc_handle: IpcHandle, metadata: dict
-) -> ContextManager[torch.TensorType]:
-    """Open a CUDA IPC handle and return a Pytorch tensor.
-
-    example use:
-    ```python
-    from dora.cuda import ipc_buffer_to_ipc_handle, open_ipc_handle
-
-    event = node.next()
-
-    ipc_handle = ipc_buffer_to_ipc_handle(event["value"], event["metadata"])
-    with open_ipc_handle(ipc_handle, event["metadata"]) as tensor:
-        pass
-    ```
-    """
-    shape = metadata["shape"]
-    strides = metadata["strides"]
-    dtype = metadata["dtype"]
-    try:
-        buffer = ipc_handle.open(get_context())
-        device_arr = DeviceNDArray(shape, strides, dtype, gpu_data=buffer)
-        yield torch.as_tensor(device_arr, device="cuda")
-    finally:
-        ipc_handle.close()

apis/python/node/generate_stubs.py

@@ -1,528 +0,0 @@
-"""TODO: Add docstring."""
-
-import argparse
-import ast
-import importlib
-import inspect
-import logging
-import re
-import subprocess
-from collections.abc import Mapping
-from functools import reduce
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
-
-
-def path_to_type(*elements: str) -> ast.AST:
-    """TODO: Add docstring."""
-    base: ast.AST = ast.Name(id=elements[0], ctx=ast.Load())
-    for e in elements[1:]:
-        base = ast.Attribute(value=base, attr=e, ctx=ast.Load())
-    return base
-
-
-OBJECT_MEMBERS = dict(inspect.getmembers(object))
-BUILTINS: Dict[str, Union[None, Tuple[List[ast.AST], ast.AST]]] = {
-    "__annotations__": None,
-    "__bool__": ([], path_to_type("bool")),
-    "__bytes__": ([], path_to_type("bytes")),
-    "__class__": None,
-    "__contains__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__del__": None,
-    "__delattr__": ([path_to_type("str")], path_to_type("None")),
-    "__delitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")),
-    "__dict__": None,
-    "__dir__": None,
-    "__doc__": None,
-    "__eq__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__format__": ([path_to_type("str")], path_to_type("str")),
-    "__ge__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__getattribute__": ([path_to_type("str")], path_to_type("typing", "Any")),
-    "__getitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")),
-    "__gt__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__hash__": ([], path_to_type("int")),
-    "__init__": ([], path_to_type("None")),
-    "__init_subclass__": None,
-    "__iter__": ([], path_to_type("typing", "Any")),
-    "__le__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__len__": ([], path_to_type("int")),
-    "__lt__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__module__": None,
-    "__ne__": ([path_to_type("typing", "Any")], path_to_type("bool")),
-    "__new__": None,
-    "__next__": ([], path_to_type("typing", "Any")),
-    "__int__": ([], path_to_type("None")),
-    "__reduce__": None,
-    "__reduce_ex__": None,
-    "__repr__": ([], path_to_type("str")),
-    "__setattr__": (
-        [path_to_type("str"), path_to_type("typing", "Any")],
-        path_to_type("None"),
-    ),
-    "__setitem__": (
-        [path_to_type("typing", "Any"), path_to_type("typing", "Any")],
-        path_to_type("typing", "Any"),
-    ),
-    "__sizeof__": None,
-    "__str__": ([], path_to_type("str")),
-    "__subclasshook__": None,
-}
-
-
-def module_stubs(module: Any) -> ast.Module:
-    """TODO: Add docstring."""
-    types_to_import = {"typing"}
-    classes = []
-    functions = []
-    for member_name, member_value in inspect.getmembers(module):
-        element_path = [module.__name__, member_name]
-        if member_name.startswith("__") or member_name.startswith("DoraStatus"):
-            pass
-        elif inspect.isclass(member_value):
-            classes.append(
-                class_stubs(member_name, member_value, element_path, types_to_import),
-            )
-        elif inspect.isbuiltin(member_value):
-            functions.append(
-                function_stub(
-                    member_name,
-                    member_value,
-                    element_path,
-                    types_to_import,
-                    in_class=False,
-                ),
-            )
-        else:
-            logging.warning(f"Unsupported root construction {member_name}")
-    return ast.Module(
-        body=[ast.Import(names=[ast.alias(name=t)]) for t in sorted(types_to_import)]
-        + classes
-        + functions,
-        type_ignores=[],
-    )
-
-
-def class_stubs(
-    cls_name: str, cls_def: Any, element_path: List[str], types_to_import: Set[str],
-) -> ast.ClassDef:
-    """TODO: Add docstring."""
-    attributes: List[ast.AST] = []
-    methods: List[ast.AST] = []
-    magic_methods: List[ast.AST] = []
-    constants: List[ast.AST] = []
-    for member_name, member_value in inspect.getmembers(cls_def):
-        current_element_path = [*element_path, member_name]
-        if member_name == "__init__":
-            try:
-                inspect.signature(cls_def)  # we check it actually exists
-                methods = [
-                    function_stub(
-                        member_name,
-                        cls_def,
-                        current_element_path,
-                        types_to_import,
-                        in_class=True,
-                    ),
-                    *methods,
-                ]
-            except ValueError as e:
-                if "no signature found" not in str(e):
-                    raise ValueError(
-                        f"Error while parsing signature of {cls_name}.__init_",
-                    ) from e
-        elif (
-            member_value == OBJECT_MEMBERS.get(member_name)
-            or BUILTINS.get(member_name, ()) is None
-        ):
-            pass
-        elif inspect.isdatadescriptor(member_value):
-            attributes.extend(
-                data_descriptor_stub(
-                    member_name, member_value, current_element_path, types_to_import,
-                ),
-            )
-        elif inspect.isroutine(member_value):
-            (magic_methods if member_name.startswith("__") else methods).append(
-                function_stub(
-                    member_name,
-                    member_value,
-                    current_element_path,
-                    types_to_import,
-                    in_class=True,
-                ),
-            )
-        elif member_name == "__match_args__":
-            constants.append(
-                ast.AnnAssign(
-                    target=ast.Name(id=member_name, ctx=ast.Store()),
-                    annotation=ast.Subscript(
-                        value=path_to_type("tuple"),
-                        slice=ast.Tuple(
-                            elts=[path_to_type("str"), ast.Ellipsis()], ctx=ast.Load(),
-                        ),
-                        ctx=ast.Load(),
-                    ),
-                    value=ast.Constant(member_value),
-                    simple=1,
-                ),
-            )
-        elif member_value is not None:
-            constants.append(
-                ast.AnnAssign(
-                    target=ast.Name(id=member_name, ctx=ast.Store()),
-                    annotation=concatenated_path_to_type(
-                        member_value.__class__.__name__, element_path, types_to_import,
-                    ),
-                    value=ast.Ellipsis(),
-                    simple=1,
-                ),
-            )
-        else:
-            logging.warning(
-                f"Unsupported member {member_name} of class {'.'.join(element_path)}",
-            )
-
-    doc = inspect.getdoc(cls_def)
-    doc_comment = build_doc_comment(doc) if doc else None
-    return ast.ClassDef(
-        cls_name,
-        bases=[],
-        keywords=[],
-        body=(
-            ([doc_comment] if doc_comment else [])
-            + attributes
-            + methods
-            + magic_methods
-            + constants
-        )
-        or [ast.Ellipsis()],
-        decorator_list=[path_to_type("typing", "final")],
-    )
-
-
-def data_descriptor_stub(
-    data_desc_name: str,
-    data_desc_def: Any,
-    element_path: List[str],
-    types_to_import: Set[str],
-) -> Union[Tuple[ast.AnnAssign, ast.Expr], Tuple[ast.AnnAssign]]:
-    """TODO: Add docstring."""
-    annotation = None
-    doc_comment = None
-
-    doc = inspect.getdoc(data_desc_def)
-    if doc is not None:
-        annotation = returns_stub(data_desc_name, doc, element_path, types_to_import)
-        m = re.findall(r"^ *:return: *(.*) *$", doc, re.MULTILINE)
-        if len(m) == 1:
-            doc_comment = m[0]
-        elif len(m) > 1:
-            raise ValueError(
-                f"Multiple return annotations found with :return: in {'.'.join(element_path)} documentation",
-            )
-
-    assign = ast.AnnAssign(
-        target=ast.Name(id=data_desc_name, ctx=ast.Store()),
-        annotation=annotation or path_to_type("typing", "Any"),
-        simple=1,
-    )
-    doc_comment = build_doc_comment(doc_comment) if doc_comment else None
-    return (assign, doc_comment) if doc_comment else (assign,)
-
-
-def function_stub(
-    fn_name: str,
-    fn_def: Any,
-    element_path: List[str],
-    types_to_import: Set[str],
-    *,
-    in_class: bool,
-) -> ast.FunctionDef:
-    """TODO: Add docstring."""
-    body: List[ast.AST] = []
-    doc = inspect.getdoc(fn_def)
-    if doc is not None:
-        doc_comment = build_doc_comment(doc)
-        if doc_comment is not None:
-            body.append(doc_comment)
-
-    decorator_list = []
-    if in_class and hasattr(fn_def, "__self__"):
-        decorator_list.append(ast.Name("staticmethod"))
-
-    return ast.FunctionDef(
-        fn_name,
-        arguments_stub(fn_name, fn_def, doc or "", element_path, types_to_import),
-        body or [ast.Ellipsis()],
-        decorator_list=decorator_list,
-        returns=(
-            returns_stub(fn_name, doc, element_path, types_to_import) if doc else None
-        ),
-        lineno=0,
-    )
-
-
-def arguments_stub(
-    callable_name: str,
-    callable_def: Any,
-    doc: str,
-    element_path: List[str],
-    types_to_import: Set[str],
-) -> ast.arguments:
-    """TODO: Add docstring."""
-    real_parameters: Mapping[str, inspect.Parameter] = inspect.signature(
-        callable_def,
-    ).parameters
-    if callable_name == "__init__":
-        real_parameters = {
-            "self": inspect.Parameter("self", inspect.Parameter.POSITIONAL_ONLY),
-            **real_parameters,
-        }
-
-    parsed_param_types = {}
-    optional_params = set()
-
-    # Types for magic functions types
-    builtin = BUILTINS.get(callable_name)
-    if isinstance(builtin, tuple):
-        param_names = list(real_parameters.keys())
-        if param_names and param_names[0] == "self":
-            del param_names[0]
-        parsed_param_types = {name: t for name, t in zip(param_names, builtin[0])}
-
-    # Types from comment
-    for match in re.findall(
-        r"^ *:type *([a-zA-Z0-9_]+): ([^\n]*) *$", doc, re.MULTILINE,
-    ):
-        if match[0] not in real_parameters:
-            raise ValueError(
-                f"The parameter {match[0]} of {'.'.join(element_path)} "
-                "is defined in the documentation but not in the function signature",
-            )
-        type = match[1]
-        if type.endswith(", optional"):
-            optional_params.add(match[0])
-            type = type[:-10]
-        parsed_param_types[match[0]] = convert_type_from_doc(
-            type, element_path, types_to_import,
-        )
-
-    # we parse the parameters
-    posonlyargs = []
-    args = []
-    vararg = None
-    kwonlyargs = []
-    kw_defaults = []
-    kwarg = None
-    defaults = []
-    for param in real_parameters.values():
-        if param.name != "self" and param.name not in parsed_param_types:
-            raise ValueError(
-                f"The parameter {param.name} of {'.'.join(element_path)} "
-                "has no type definition in the function documentation",
-            )
-        param_ast = ast.arg(
-            arg=param.name, annotation=parsed_param_types.get(param.name),
-        )
-
-        default_ast = None
-        if param.default != param.empty:
-            default_ast = ast.Constant(param.default)
-            if param.name not in optional_params:
-                raise ValueError(
-                    f"Parameter {param.name} of {'.'.join(element_path)} "
-                    "is optional according to the type but not flagged as such in the doc",
-                )
-        elif param.name in optional_params:
-            raise ValueError(
-                f"Parameter {param.name} of {'.'.join(element_path)} "
-                "is optional according to the documentation but has no default value",
-            )
-
-        if param.kind == param.POSITIONAL_ONLY:
-            args.append(param_ast)
-            # posonlyargs.append(param_ast)
-            # defaults.append(default_ast)
-        elif param.kind == param.POSITIONAL_OR_KEYWORD:
-            args.append(param_ast)
-            defaults.append(default_ast)
-        elif param.kind == param.VAR_POSITIONAL:
-            vararg = param_ast
-        elif param.kind == param.KEYWORD_ONLY:
-            kwonlyargs.append(param_ast)
-            kw_defaults.append(default_ast)
-        elif param.kind == param.VAR_KEYWORD:
-            kwarg = param_ast
-
-    return ast.arguments(
-        posonlyargs=posonlyargs,
-        args=args,
-        vararg=vararg,
-        kwonlyargs=kwonlyargs,
-        kw_defaults=kw_defaults,
-        defaults=defaults,
-        kwarg=kwarg,
-    )
-
-
-def returns_stub(
-    callable_name: str, doc: str, element_path: List[str], types_to_import: Set[str],
-) -> Optional[ast.AST]:
-    """TODO: Add docstring."""
-    m = re.findall(r"^ *:rtype: *([^\n]*) *$", doc, re.MULTILINE)
-    if len(m) == 0:
-        builtin = BUILTINS.get(callable_name)
-        if isinstance(builtin, tuple) and builtin[1] is not None:
-            return builtin[1]
-        raise ValueError(
-            f"The return type of {'.'.join(element_path)} "
-            "has no type definition using :rtype: in the function documentation",
-        )
-    if len(m) > 1:
-        raise ValueError(
-            f"Multiple return type annotations found with :rtype: for {'.'.join(element_path)}",
-        )
-    return convert_type_from_doc(m[0], element_path, types_to_import)
-
-
-def convert_type_from_doc(
-    type_str: str, element_path: List[str], types_to_import: Set[str],
-) -> ast.AST:
-    """TODO: Add docstring."""
-    type_str = type_str.strip()
-    return parse_type_to_ast(type_str, element_path, types_to_import)
-
-
-def parse_type_to_ast(
-    type_str: str, element_path: List[str], types_to_import: Set[str],
-) -> ast.AST:
-    # let's tokenize
-    """TODO: Add docstring."""
-    tokens = []
-    current_token = ""
-    for c in type_str:
-        if "a" <= c <= "z" or "A" <= c <= "Z" or c == ".":
-            current_token += c
-        else:
-            if current_token:
-                tokens.append(current_token)
-            current_token = ""
-            if c != " ":
-                tokens.append(c)
-    if current_token:
-        tokens.append(current_token)
-
-    # let's first parse nested parenthesis
-    stack: List[List[Any]] = [[]]
-    for token in tokens:
-        if token == "[":
-            children: List[str] = []
-            stack[-1].append(children)
-            stack.append(children)
-        elif token == "]":
-            stack.pop()
-        else:
-            stack[-1].append(token)
-
-    # then it's easy
-    def parse_sequence(sequence: List[Any]) -> ast.AST:
-        # we split based on "or"
-        """TODO: Add docstring."""
-        or_groups: List[List[str]] = [[]]
-        print(sequence)
-        # TODO: Fix sequence
-        if ("Ros" in sequence and "2" in sequence) or ("dora.Ros" in sequence and "2" in sequence):
-            sequence = ["".join(sequence)]
-
-        for e in sequence:
-            if e == "or":
-                or_groups.append([])
-            else:
-                or_groups[-1].append(e)
-        if any(not g for g in or_groups):
-            raise ValueError(
-                f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}",
-            )
-
-        new_elements: List[ast.AST] = []
-        for group in or_groups:
-            if len(group) == 1 and isinstance(group[0], str):
-                new_elements.append(
-                    concatenated_path_to_type(group[0], element_path, types_to_import),
-                )
-            elif (
-                len(group) == 2
-                and isinstance(group[0], str)
-                and isinstance(group[1], list)
-            ):
-                new_elements.append(
-                    ast.Subscript(
-                        value=concatenated_path_to_type(
-                            group[0], element_path, types_to_import,
-                        ),
-                        slice=parse_sequence(group[1]),
-                        ctx=ast.Load(),
-                    ),
-                )
-            else:
-                raise ValueError(
-                    f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}",
-                )
-        return reduce(
-            lambda left, right: ast.BinOp(left=left, op=ast.BitOr(), right=right),
-            new_elements,
-        )
-
-    return parse_sequence(stack[0])
-
-
-def concatenated_path_to_type(
-    path: str, element_path: List[str], types_to_import: Set[str],
-) -> ast.AST:
-    """TODO: Add docstring."""
-    parts = path.split(".")
-    if any(not p for p in parts):
-        raise ValueError(
-            f"Not able to parse type '{path}' used by {'.'.join(element_path)}",
-        )
-    if len(parts) > 1:
-        types_to_import.add(".".join(parts[:-1]))
-    return path_to_type(*parts)
-
-
-def build_doc_comment(doc: str) -> Optional[ast.Expr]:
-    """TODO: Add docstring."""
-    lines = [line.strip() for line in doc.split("\n")]
-    clean_lines = []
-    for line in lines:
-        if line.startswith((":type", ":rtype")):
-            continue
-        clean_lines.append(line)
-    text = "\n".join(clean_lines).strip()
-    return ast.Expr(value=ast.Constant(text)) if text else None
-
-
-def format_with_ruff(file: str) -> None:
-    """TODO: Add docstring."""
-    subprocess.check_call(["python", "-m", "ruff", "format", file])
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Extract Python type stub from a python module.",
-    )
-    parser.add_argument(
-        "module_name", help="Name of the Python module for which generate stubs",
-    )
-    parser.add_argument(
-        "out",
-        help="Name of the Python stub file to write to",
-        type=argparse.FileType("wt"),
-    )
-    parser.add_argument(
-        "--ruff", help="Formats the generated stubs using Ruff", action="store_true",
-    )
-    args = parser.parse_args()
-    stub_content = ast.unparse(module_stubs(importlib.import_module(args.module_name)))
-    args.out.write(stub_content)
-    if args.ruff:
-        format_with_ruff(args.out.name)

apis/python/node/pyproject.toml

@@ -1,24 +0,0 @@
-[build-system]
-requires = ["maturin>=0.13.2"]
-build-backend = "maturin"
-
-[project]
-name = "dora-rs"
-dynamic = ["version"]
-# Install pyarrow at the same time of dora-rs
-requires-python = ">=3.7"
-license = { text = "MIT" }
-readme = "README.md"
-dependencies = ['pyarrow']
-
-[dependency-groups]
-dev = ["pytest >=7.1.1", "ruff >=0.9.1"]
-
-[tool.maturin]
-features = ["pyo3/extension-module"]
-
-[tool.ruff.lint]
-extend-select = [
-    "D",  # pydocstyle
-    "UP",
-]

apis/python/node/src/lib.rs

@@ -1,399 +0,0 @@
-#![allow(clippy::borrow_deref_ref)] // clippy warns about code generated by #[pymethods]
-
-use std::env::current_dir;
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Duration;
-
-use arrow::pyarrow::{FromPyArrow, ToPyArrow};
-use dora_download::download_file;
-use dora_node_api::dora_core::config::NodeId;
-use dora_node_api::dora_core::descriptor::source_is_url;
-use dora_node_api::merged::{MergeExternalSend, MergedEvent};
-use dora_node_api::{DataflowId, DoraNode, EventStream};
-use dora_operator_api_python::{DelayedCleanup, NodeCleanupHandle, PyEvent, pydict_to_metadata};
-use dora_ros2_bridge_python::Ros2Subscription;
-use eyre::Context;
-use futures::{Stream, StreamExt};
-use pyo3::prelude::*;
-use pyo3::types::{PyBytes, PyDict};
-use pyo3_special_method_derive::{Dict, Dir, Repr, Str};
-
-/// The custom node API lets you integrate `dora` into your application.
-/// It allows you to retrieve input and send output in any fashion you want.
-///
-/// Use with:
-///
-/// ```python
-/// from dora import Node
-///
-/// node = Node()
-/// ```
-///
-/// :type node_id: str, optional
-#[pyclass]
-#[derive(Dir, Dict, Str, Repr)]
-pub struct Node {
-    events: Events,
-    node: DelayedCleanup<DoraNode>,
-
-    dataflow_id: DataflowId,
-    node_id: NodeId,
-}
-
-#[pymethods]
-impl Node {
-    #[new]
-    #[pyo3(signature = (node_id=None))]
-    pub fn new(node_id: Option<String>) -> eyre::Result<Self> {
-        let (node, events) = if let Some(node_id) = node_id {
-            DoraNode::init_flexible(NodeId::from(node_id))
-                .context("Could not setup node from node id. Make sure to have a running dataflow with this dynamic node")?
-        } else {
-            DoraNode::init_from_env().context("Could not initiate node from environment variable. For dynamic node, please add a node id in the initialization function.")?
-        };
-
-        let dataflow_id = *node.dataflow_id();
-        let node_id = node.id().clone();
-        let node = DelayedCleanup::new(node);
-        let events = events;
-        let cleanup_handle = NodeCleanupHandle {
-            _handles: Arc::new(node.handle()),
-        };
-        Ok(Node {
-            events: Events {
-                inner: EventsInner::Dora(events),
-                _cleanup_handle: cleanup_handle,
-            },
-            dataflow_id,
-            node_id,
-            node,
-        })
-    }
-
-    /// `.next()` gives you the next input that the node has received.
-    /// It blocks until the next event becomes available.
-    /// You can use timeout in seconds to return if no input is available.
-    /// It will return `None` when all senders has been dropped.
-    ///
-    /// ```python
-    /// event = node.next()
-    /// ```
-    ///
-    /// You can also iterate over the event stream with a loop
-    ///
-    /// ```python
-    /// for event in node:
-    ///    match event["type"]:
-    ///        case "INPUT":
-    ///            match event["id"]:
-    ///                 case "image":
-    /// ```
-    ///
-    /// :type timeout: float, optional
-    /// :rtype: dict
-    #[pyo3(signature = (timeout=None))]
-    #[allow(clippy::should_implement_trait)]
-    pub fn next(&mut self, py: Python, timeout: Option<f32>) -> PyResult<Option<Py<PyDict>>> {
-        let event = py.allow_threads(|| self.events.recv(timeout.map(Duration::from_secs_f32)));
-        if let Some(event) = event {
-            let dict = event
-                .to_py_dict(py)
-                .context("Could not convert event into a dict")?;
-            Ok(Some(dict))
-        } else {
-            Ok(None)
-        }
-    }
-
-    /// `.recv_async()` gives you the next input that the node has received asynchronously.
-    /// It does not blocks until the next event becomes available.
-    /// You can use timeout in seconds to return if no input is available.
-    /// It will return an Error if the timeout is reached.
-    /// It will return `None` when all senders has been dropped.
-    ///
-    /// warning::
-    ///     This feature is experimental as pyo3 async (rust-python FFI) is still in development.
-    ///
-    /// ```python
-    /// event = await node.recv_async()
-    /// ```
-    ///
-    /// You can also iterate over the event stream with a loop
-    ///
-    /// :type timeout: float, optional
-    /// :rtype: dict
-    #[pyo3(signature = (timeout=None))]
-    #[allow(clippy::should_implement_trait)]
-    pub async fn recv_async(&mut self, timeout: Option<f32>) -> PyResult<Option<Py<PyDict>>> {
-        let event = self
-            .events
-            .recv_async_timeout(timeout.map(Duration::from_secs_f32))
-            .await;
-        if let Some(event) = event {
-            // Get python
-            Python::with_gil(|py| {
-                let dict = event
-                    .to_py_dict(py)
-                    .context("Could not convert event into a dict")?;
-                Ok(Some(dict))
-            })
-        } else {
-            Ok(None)
-        }
-    }
-
-    /// You can iterate over the event stream with a loop
-    ///
-    /// ```python
-    /// for event in node:
-    ///    match event["type"]:
-    ///        case "INPUT":
-    ///            match event["id"]:
-    ///                 case "image":
-    /// ```
-    ///
-    /// Default behaviour is to timeout after 2 seconds.
-    ///
-    /// :rtype: dict
-    pub fn __next__(&mut self, py: Python) -> PyResult<Option<Py<PyDict>>> {
-        self.next(py, None)
-    }
-
-    /// You can iterate over the event stream with a loop
-    ///
-    /// ```python
-    /// for event in node:
-    ///    match event["type"]:
-    ///        case "INPUT":
-    ///            match event["id"]:
-    ///                 case "image":
-    /// ```
-    ///
-    /// :rtype: dict
-    fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
-        slf
-    }
-
-    /// `send_output` send data from the node.
-    ///
-    /// ```python
-    /// Args:
-    ///    output_id: str,
-    ///    data: pyarrow.Array,
-    ///    metadata: Option[Dict],
-    /// ```
-    ///
-    /// ex:
-    ///
-    /// ```python
-    /// node.send_output("string", b"string", {"open_telemetry_context": "7632e76"})
-    /// ```
-    ///
-    /// :type output_id: str
-    /// :type data: pyarrow.Array
-    /// :type metadata: dict, optional
-    /// :rtype: None
-    #[pyo3(signature = (output_id, data, metadata=None))]
-    pub fn send_output(
-        &mut self,
-        output_id: String,
-        data: PyObject,
-        metadata: Option<Bound<'_, PyDict>>,
-        py: Python,
-    ) -> eyre::Result<()> {
-        let parameters = pydict_to_metadata(metadata)?;
-
-        if let Ok(py_bytes) = data.downcast_bound::<PyBytes>(py) {
-            let data = py_bytes.as_bytes();
-            self.node
-                .get_mut()
-                .send_output_bytes(output_id.into(), parameters, data.len(), data)
-                .wrap_err("failed to send output")?;
-        } else if let Ok(arrow_array) = arrow::array::ArrayData::from_pyarrow_bound(data.bind(py)) {
-            self.node.get_mut().send_output(
-                output_id.into(),
-                parameters,
-                arrow::array::make_array(arrow_array),
-            )?;
-        } else {
-            eyre::bail!("invalid `data` type, must by `PyBytes` or arrow array")
-        }
-
-        Ok(())
-    }
-
-    /// Returns the full dataflow descriptor that this node is part of.
-    ///
-    /// This method returns the parsed dataflow YAML file.
-    ///
-    /// :rtype: dict
-    pub fn dataflow_descriptor(&mut self, py: Python) -> eyre::Result<PyObject> {
-        Ok(
-            pythonize::pythonize(py, &self.node.get_mut().dataflow_descriptor()?)
-                .map(|x| x.unbind())?,
-        )
-    }
-
-    /// Returns the dataflow id.
-    ///
-    /// :rtype: str
-    pub fn dataflow_id(&self) -> String {
-        self.dataflow_id.to_string()
-    }
-
-    /// Merge an external event stream with dora main loop.
-    /// This currently only work with ROS2.
-    ///
-    /// :type subscription: dora.Ros2Subscription
-    /// :rtype: None
-    pub fn merge_external_events(
-        &mut self,
-        subscription: &mut Ros2Subscription,
-    ) -> eyre::Result<()> {
-        let subscription = subscription.into_stream()?;
-        let stream = futures::stream::poll_fn(move |cx| {
-            let s = subscription.as_stream().map(|item| {
-                match item.context("failed to read ROS2 message") {
-                    Ok((value, _info)) => Python::with_gil(|py| {
-                        value
-                            .to_pyarrow(py)
-                            .context("failed to convert value to pyarrow")
-                            .unwrap_or_else(|err| err_to_pyany(err, py))
-                    }),
-                    Err(err) => Python::with_gil(|py| err_to_pyany(err, py)),
-                }
-            });
-            futures::pin_mut!(s);
-            s.poll_next_unpin(cx)
-        });
-
-        // take out the event stream and temporarily replace it with a dummy
-        let events = std::mem::replace(
-            &mut self.events.inner,
-            EventsInner::Merged(Box::new(futures::stream::empty())),
-        );
-        // update self.events with the merged stream
-        self.events.inner = EventsInner::Merged(events.merge_external_send(Box::pin(stream)));
-
-        Ok(())
-    }
-}
-
-fn err_to_pyany(err: eyre::Report, gil: Python<'_>) -> Py<PyAny> {
-    PyErr::from(err)
-        .into_pyobject(gil)
-        .unwrap_or_else(|infallible| match infallible {})
-        .into_any()
-        .unbind()
-}
-
-struct Events {
-    inner: EventsInner,
-    _cleanup_handle: NodeCleanupHandle,
-}
-
-impl Events {
-    fn recv(&mut self, timeout: Option<Duration>) -> Option<PyEvent> {
-        let event = match &mut self.inner {
-            EventsInner::Dora(events) => match timeout {
-                Some(timeout) => events.recv_timeout(timeout).map(MergedEvent::Dora),
-                None => events.recv().map(MergedEvent::Dora),
-            },
-            EventsInner::Merged(events) => futures::executor::block_on(events.next()),
-        };
-        event.map(|event| PyEvent { event })
-    }
-
-    async fn recv_async_timeout(&mut self, timeout: Option<Duration>) -> Option<PyEvent> {
-        let event = match &mut self.inner {
-            EventsInner::Dora(events) => match timeout {
-                Some(timeout) => events
-                    .recv_async_timeout(timeout)
-                    .await
-                    .map(MergedEvent::Dora),
-                None => events.recv_async().await.map(MergedEvent::Dora),
-            },
-            EventsInner::Merged(events) => events.next().await,
-        };
-        event.map(|event| PyEvent { event })
-    }
-}
-
-#[allow(clippy::large_enum_variant)]
-enum EventsInner {
-    Dora(EventStream),
-    Merged(Box<dyn Stream<Item = MergedEvent<PyObject>> + Unpin + Send + Sync>),
-}
-
-impl<'a> MergeExternalSend<'a, PyObject> for EventsInner {
-    type Item = MergedEvent<PyObject>;
-
-    fn merge_external_send(
-        self,
-        external_events: impl Stream<Item = PyObject> + Unpin + Send + Sync + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + Send + Sync + 'a> {
-        match self {
-            EventsInner::Dora(events) => events.merge_external_send(external_events),
-            EventsInner::Merged(events) => {
-                let merged = events.merge_external_send(external_events);
-                Box::new(merged.map(|event| match event {
-                    MergedEvent::Dora(e) => MergedEvent::Dora(e),
-                    MergedEvent::External(e) => MergedEvent::External(e.flatten()),
-                }))
-            }
-        }
-    }
-}
-
-impl Node {
-    pub fn id(&self) -> String {
-        self.node_id.to_string()
-    }
-}
-
-/// Start a runtime for Operators
-///
-/// :rtype: None
-#[pyfunction]
-pub fn start_runtime() -> eyre::Result<()> {
-    dora_runtime::main().wrap_err("Dora Runtime raised an error.")
-}
-
-pub fn resolve_dataflow(dataflow: String) -> eyre::Result<PathBuf> {
-    let dataflow = if source_is_url(&dataflow) {
-        // try to download the shared library
-        let target_path = current_dir().context("Could not access the current dir")?;
-        let rt = tokio::runtime::Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .context("tokio runtime failed")?;
-        rt.block_on(async { download_file(&dataflow, &target_path).await })
-            .wrap_err("failed to download dataflow yaml file")?
-    } else {
-        PathBuf::from(dataflow)
-    };
-    Ok(dataflow)
-}
-
-/// Run a Dataflow
-///
-/// :rtype: None
-#[pyfunction]
-#[pyo3(signature = (dataflow_path, uv=None))]
-pub fn run(dataflow_path: String, uv: Option<bool>) -> eyre::Result<()> {
-    dora_cli::run_func(dataflow_path, uv.unwrap_or_default())
-}
-
-#[pymodule]
-fn dora(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
-    dora_ros2_bridge_python::create_dora_ros2_bridge_module(&m)?;
-
-    m.add_function(wrap_pyfunction!(start_runtime, &m)?)?;
-    m.add_function(wrap_pyfunction!(run, &m)?)?;
-    m.add_class::<Node>()?;
-    m.setattr("__version__", env!("CARGO_PKG_VERSION"))?;
-    m.setattr("__author__", "Dora-rs Authors")?;
-
-    Ok(())
-}

apis/python/operator/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "dora-operator-api-python"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-dora-node-api = { workspace = true }
-pyo3 = { workspace = true, features = ["eyre", "abi3-py37"] }
-eyre = "0.6"
-serde_yaml = { workspace = true }
-flume = "0.10.14"
-arrow = { workspace = true, features = ["pyarrow"] }
-arrow-schema = { workspace = true }
-aligned-vec = "0.5.0"
-futures = "0.3.28"
-futures-concurrency = "7.3.0"

apis/python/operator/src/lib.rs

@@ -1,373 +0,0 @@
-use std::{
-    collections::{BTreeMap, HashMap},
-    sync::{Arc, Mutex},
-};
-
-use arrow::pyarrow::ToPyArrow;
-use dora_node_api::{
-    DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, StopCause,
-    merged::{MergeExternalSend, MergedEvent},
-};
-use eyre::{Context, Result};
-use futures::{Stream, StreamExt};
-use futures_concurrency::stream::Merge as _;
-use pyo3::{
-    prelude::*,
-    types::{IntoPyDict, PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTuple},
-};
-
-/// Dora Event
-pub struct PyEvent {
-    pub event: MergedEvent<PyObject>,
-}
-
-/// Keeps the dora node alive until all event objects have been dropped.
-#[derive(Clone)]
-#[pyclass]
-pub struct NodeCleanupHandle {
-    pub _handles: Arc<CleanupHandle<DoraNode>>,
-}
-
-/// Owned type with delayed cleanup (using `handle` method).
-pub struct DelayedCleanup<T>(Arc<Mutex<T>>);
-
-impl<T> DelayedCleanup<T> {
-    pub fn new(value: T) -> Self {
-        Self(Arc::new(Mutex::new(value)))
-    }
-
-    pub fn handle(&self) -> CleanupHandle<T> {
-        CleanupHandle(self.0.clone())
-    }
-
-    pub fn get_mut(&mut self) -> std::sync::MutexGuard<T> {
-        self.0.try_lock().expect("failed to lock DelayedCleanup")
-    }
-}
-
-impl Stream for DelayedCleanup<EventStream> {
-    type Item = Event;
-
-    fn poll_next(
-        self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        let mut inner: std::sync::MutexGuard<'_, EventStream> = self.get_mut().get_mut();
-        inner.poll_next_unpin(cx)
-    }
-}
-
-impl<'a, E> MergeExternalSend<'a, E> for DelayedCleanup<EventStream>
-where
-    E: 'static,
-{
-    type Item = MergedEvent<E>;
-
-    fn merge_external_send(
-        self,
-        external_events: impl Stream<Item = E> + Unpin + Send + Sync + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + Send + Sync + 'a> {
-        let dora = self.map(MergedEvent::Dora);
-        let external = external_events.map(MergedEvent::External);
-        Box::new((dora, external).merge())
-    }
-}
-
-#[allow(dead_code)]
-pub struct CleanupHandle<T>(Arc<Mutex<T>>);
-
-impl PyEvent {
-    pub fn to_py_dict(self, py: Python<'_>) -> PyResult<Py<PyDict>> {
-        let mut pydict = HashMap::new();
-        match &self.event {
-            MergedEvent::Dora(_) => pydict.insert(
-                "kind",
-                "dora"
-                    .into_pyobject(py)
-                    .context("Failed to create pystring")?
-                    .unbind()
-                    .into(),
-            ),
-            MergedEvent::External(_) => pydict.insert(
-                "kind",
-                "external"
-                    .into_pyobject(py)
-                    .context("Failed to create pystring")?
-                    .unbind()
-                    .into(),
-            ),
-        };
-        match &self.event {
-            MergedEvent::Dora(event) => {
-                if let Some(id) = Self::id(event) {
-                    pydict.insert(
-                        "id",
-                        id.into_pyobject(py)
-                            .context("Failed to create id pyobject")?
-                            .into(),
-                    );
-                }
-                pydict.insert(
-                    "type",
-                    Self::ty(event)
-                        .into_pyobject(py)
-                        .context("Failed to create event pyobject")?
-                        .unbind()
-                        .into(),
-                );
-
-                if let Some(value) = self.value(py)? {
-                    pydict.insert("value", value);
-                }
-                if let Some(metadata) = Self::metadata(event, py)? {
-                    pydict.insert("metadata", metadata);
-                }
-                if let Some(error) = Self::error(event) {
-                    pydict.insert(
-                        "error",
-                        error
-                            .into_pyobject(py)
-                            .context("Failed to create error pyobject")?
-                            .unbind()
-                            .into(),
-                    );
-                }
-            }
-            MergedEvent::External(event) => {
-                pydict.insert("value", event.clone_ref(py));
-            }
-        }
-
-        Ok(pydict
-            .into_py_dict(py)
-            .context("Failed to create py_dict")?
-            .unbind())
-    }
-
-    fn ty(event: &Event) -> &str {
-        match event {
-            Event::Stop(_) => "STOP",
-            Event::Input { .. } => "INPUT",
-            Event::InputClosed { .. } => "INPUT_CLOSED",
-            Event::Error(_) => "ERROR",
-            _other => "UNKNOWN",
-        }
-    }
-
-    fn id(event: &Event) -> Option<&str> {
-        match event {
-            Event::Input { id, .. } => Some(id),
-            Event::InputClosed { id } => Some(id),
-            Event::Stop(cause) => match cause {
-                StopCause::Manual => Some("MANUAL"),
-                StopCause::AllInputsClosed => Some("ALL_INPUTS_CLOSED"),
-                &_ => None,
-            },
-            _ => None,
-        }
-    }
-
-    /// Returns the payload of an input event as an arrow array (if any).
-    fn value(&self, py: Python<'_>) -> PyResult<Option<PyObject>> {
-        match &self.event {
-            MergedEvent::Dora(Event::Input { data, .. }) => {
-                // TODO: Does this call leak data?&
-                let array_data = data.to_data().to_pyarrow(py)?;
-                Ok(Some(array_data))
-            }
-            _ => Ok(None),
-        }
-    }
-
-    fn metadata(event: &Event, py: Python<'_>) -> Result<Option<PyObject>> {
-        match event {
-            Event::Input { metadata, .. } => Ok(Some(
-                metadata_to_pydict(metadata, py)
-                    .context("Issue deserializing metadata")?
-                    .into_pyobject(py)
-                    .context("Failed to create metadata_to_pydice")?
-                    .unbind()
-                    .into(),
-            )),
-            _ => Ok(None),
-        }
-    }
-
-    fn error(event: &Event) -> Option<&str> {
-        match event {
-            Event::Error(error) => Some(error),
-            _other => None,
-        }
-    }
-}
-
-pub fn pydict_to_metadata(dict: Option<Bound<'_, PyDict>>) -> Result<MetadataParameters> {
-    let mut parameters = BTreeMap::default();
-    if let Some(pymetadata) = dict {
-        for (key, value) in pymetadata.iter() {
-            let key = key.extract::<String>().context("Parsing metadata keys")?;
-            if value.is_exact_instance_of::<PyBool>() {
-                parameters.insert(key, Parameter::Bool(value.extract()?))
-            } else if value.is_instance_of::<PyInt>() {
-                parameters.insert(key, Parameter::Integer(value.extract::<i64>()?))
-            } else if value.is_instance_of::<PyFloat>() {
-                parameters.insert(key, Parameter::Float(value.extract::<f64>()?))
-            } else if value.is_instance_of::<PyString>() {
-                parameters.insert(key, Parameter::String(value.extract()?))
-            } else if (value.is_instance_of::<PyTuple>() || value.is_instance_of::<PyList>())
-                && value.len()? > 0
-                && value.get_item(0)?.is_exact_instance_of::<PyInt>()
-            {
-                let list: Vec<i64> = value.extract()?;
-                parameters.insert(key, Parameter::ListInt(list))
-            } else if (value.is_instance_of::<PyTuple>() || value.is_instance_of::<PyList>())
-                && value.len()? > 0
-                && value.get_item(0)?.is_exact_instance_of::<PyFloat>()
-            {
-                let list: Vec<f64> = value.extract()?;
-                parameters.insert(key, Parameter::ListFloat(list))
-            } else if value.is_instance_of::<PyList>()
-                && value.len()? > 0
-                && value.get_item(0)?.is_exact_instance_of::<PyString>()
-            {
-                let list: Vec<String> = value.extract()?;
-                parameters.insert(key, Parameter::ListString(list))
-            } else {
-                println!("could not convert type {value}");
-                parameters.insert(key, Parameter::String(value.str()?.to_string()))
-            };
-        }
-    }
-    Ok(parameters)
-}
-
-pub fn metadata_to_pydict<'a>(
-    metadata: &'a Metadata,
-    py: Python<'a>,
-) -> Result<pyo3::Bound<'a, PyDict>> {
-    let dict = PyDict::new(py);
-    for (k, v) in metadata.parameters.iter() {
-        match v {
-            Parameter::Bool(bool) => dict
-                .set_item(k, bool)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::Integer(int) => dict
-                .set_item(k, int)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::Float(float) => dict
-                .set_item(k, float)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::String(s) => dict
-                .set_item(k, s)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::ListInt(l) => dict
-                .set_item(k, l)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::ListFloat(l) => dict
-                .set_item(k, l)
-                .context("Could not insert metadata into python dictionary")?,
-            Parameter::ListString(l) => dict
-                .set_item(k, l)
-                .context("Could not insert metadata into python dictionary")?,
-        }
-    }
-
-    Ok(dict)
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{ptr::NonNull, sync::Arc};
-
-    use aligned_vec::{AVec, ConstAlign};
-    use arrow::{
-        array::{
-            ArrayData, ArrayRef, BooleanArray, Float64Array, Int8Array, Int32Array, Int64Array,
-            ListArray, StructArray,
-        },
-        buffer::Buffer,
-    };
-
-    use arrow_schema::{DataType, Field};
-    use dora_node_api::arrow_utils::{
-        buffer_into_arrow_array, copy_array_into_sample, required_data_size,
-    };
-    use eyre::{Context, Result};
-
-    fn assert_roundtrip(arrow_array: &ArrayData) -> Result<()> {
-        let size = required_data_size(arrow_array);
-        let mut sample: AVec<u8, ConstAlign<128>> = AVec::__from_elem(128, 0, size);
-
-        let info = copy_array_into_sample(&mut sample, arrow_array);
-
-        let serialized_deserialized_arrow_array = {
-            let ptr = NonNull::new(sample.as_ptr() as *mut _).unwrap();
-            let len = sample.len();
-
-            let raw_buffer = unsafe {
-                arrow::buffer::Buffer::from_custom_allocation(ptr, len, Arc::new(sample))
-            };
-            buffer_into_arrow_array(&raw_buffer, &info)?
-        };
-
-        assert_eq!(arrow_array, &serialized_deserialized_arrow_array);
-
-        Ok(())
-    }
-
-    #[test]
-    fn serialize_deserialize_arrow() -> Result<()> {
-        // Int8
-        let arrow_array = Int8Array::from(vec![1, -2, 3, 4]).into();
-        assert_roundtrip(&arrow_array).context("Int8Array roundtrip failed")?;
-
-        // Int64
-        let arrow_array = Int64Array::from(vec![1, -2, 3, 4]).into();
-        assert_roundtrip(&arrow_array).context("Int64Array roundtrip failed")?;
-
-        // Float64
-        let arrow_array = Float64Array::from(vec![1., -2., 3., 4.]).into();
-        assert_roundtrip(&arrow_array).context("Float64Array roundtrip failed")?;
-
-        // Struct
-        let boolean = Arc::new(BooleanArray::from(vec![false, false, true, true]));
-        let int = Arc::new(Int32Array::from(vec![42, 28, 19, 31]));
-
-        let struct_array = StructArray::from(vec![
-            (
-                Arc::new(Field::new("b", DataType::Boolean, false)),
-                boolean as ArrayRef,
-            ),
-            (
-                Arc::new(Field::new("c", DataType::Int32, false)),
-                int as ArrayRef,
-            ),
-        ])
-        .into();
-        assert_roundtrip(&struct_array).context("StructArray roundtrip failed")?;
-
-        // List
-        let value_data = ArrayData::builder(DataType::Int32)
-            .len(8)
-            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
-            .build()
-            .unwrap();
-
-        // Construct a buffer for value offsets, for the nested array:
-        //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-        let value_offsets = Buffer::from_slice_ref([0, 3, 6, 8]);
-
-        // Construct a list array from the above two
-        let list_data_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, false)));
-        let list_data = ArrayData::builder(list_data_type)
-            .len(3)
-            .add_buffer(value_offsets)
-            .add_child_data(value_data)
-            .build()
-            .unwrap();
-        let list_array = ListArray::from(list_data).into();
-        assert_roundtrip(&list_array).context("ListArray roundtrip failed")?;
-
-        Ok(())
-    }
-}

apis/rust/node/Cargo.toml

@@ -1,35 +0,0 @@
-[package]
-name = "dora-node-api"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-[features]
-default = ["tracing", "metrics"]
-tracing = ["dep:dora-tracing"]
-metrics = ["dep:dora-metrics"]
-
-[dependencies]
-dora-core = { workspace = true }
-dora-message = { workspace = true }
-shared-memory-server = { workspace = true }
-eyre = "0.6.7"
-serde_yaml = { workspace = true }
-tracing = "0.1.33"
-flume = "0.10.14"
-bincode = "1.3.3"
-shared_memory_extended = "0.13.0"
-dora-tracing = { workspace = true, optional = true }
-dora-metrics = { workspace = true, optional = true }
-arrow = { workspace = true }
-futures = "0.3.28"
-futures-concurrency = "7.3.0"
-futures-timer = "3.0.2"
-dora-arrow-convert = { workspace = true }
-aligned-vec = "0.5.0"
-serde_json = "1.0.86"
-tokio = { version = "1.24.2", features = ["rt", "rt-multi-thread"] }

apis/rust/node/src/daemon_connection/mod.rs

@@ -1,86 +0,0 @@
-use dora_core::{config::NodeId, uhlc::Timestamp};
-use dora_message::{
-    DataflowId,
-    daemon_to_node::DaemonReply,
-    node_to_daemon::{DaemonRequest, NodeRegisterRequest, Timestamped},
-};
-use eyre::{Context, bail, eyre};
-use shared_memory_server::{ShmemClient, ShmemConf};
-#[cfg(unix)]
-use std::os::unix::net::UnixStream;
-use std::{
-    net::{SocketAddr, TcpStream},
-    time::Duration,
-};
-
-mod tcp;
-#[cfg(unix)]
-mod unix_domain;
-
-pub enum DaemonChannel {
-    Shmem(ShmemClient<Timestamped<DaemonRequest>, DaemonReply>),
-    Tcp(TcpStream),
-    #[cfg(unix)]
-    UnixDomain(UnixStream),
-}
-
-impl DaemonChannel {
-    #[tracing::instrument(level = "trace")]
-    pub fn new_tcp(socket_addr: SocketAddr) -> eyre::Result<Self> {
-        let stream = TcpStream::connect(socket_addr).wrap_err("failed to open TCP connection")?;
-        stream.set_nodelay(true).context("failed to set nodelay")?;
-        Ok(DaemonChannel::Tcp(stream))
-    }
-
-    #[tracing::instrument(level = "trace")]
-    pub unsafe fn new_shmem(daemon_control_region_id: &str) -> eyre::Result<Self> {
-        let daemon_events_region = ShmemConf::new()
-            .os_id(daemon_control_region_id)
-            .open()
-            .wrap_err("failed to connect to dora-daemon")?;
-        let channel = DaemonChannel::Shmem(
-            unsafe { ShmemClient::new(daemon_events_region, Some(Duration::from_secs(5))) }
-                .wrap_err("failed to create ShmemChannel")?,
-        );
-        Ok(channel)
-    }
-
-    #[cfg(unix)]
-    #[tracing::instrument(level = "trace")]
-    pub fn new_unix_socket(path: &std::path::PathBuf) -> eyre::Result<Self> {
-        let stream = UnixStream::connect(path).wrap_err("failed to open Unix socket")?;
-        Ok(DaemonChannel::UnixDomain(stream))
-    }
-
-    pub fn register(
-        &mut self,
-        dataflow_id: DataflowId,
-        node_id: NodeId,
-        timestamp: Timestamp,
-    ) -> eyre::Result<()> {
-        let msg = Timestamped {
-            inner: DaemonRequest::Register(NodeRegisterRequest::new(dataflow_id, node_id)),
-            timestamp,
-        };
-        let reply = self
-            .request(&msg)
-            .wrap_err("failed to send register request to dora-daemon")?;
-
-        match reply {
-            DaemonReply::Result(result) => result
-                .map_err(|e| eyre!(e))
-                .wrap_err("failed to register node with dora-daemon")?,
-            other => bail!("unexpected register reply: {other:?}"),
-        }
-        Ok(())
-    }
-
-    pub fn request(&mut self, request: &Timestamped<DaemonRequest>) -> eyre::Result<DaemonReply> {
-        match self {
-            DaemonChannel::Shmem(client) => client.request(request),
-            DaemonChannel::Tcp(stream) => tcp::request(stream, request),
-            #[cfg(unix)]
-            DaemonChannel::UnixDomain(stream) => unix_domain::request(stream, request),
-        }
-    }
-}

apis/rust/node/src/daemon_connection/tcp.rs

@@ -1,86 +0,0 @@
-use dora_message::{
-    daemon_to_node::DaemonReply,
-    node_to_daemon::{DaemonRequest, Timestamped},
-};
-use eyre::{Context, eyre};
-use std::{
-    io::{Read, Write},
-    net::TcpStream,
-};
-
-enum Serializer {
-    Bincode,
-    SerdeJson,
-}
-pub fn request(
-    connection: &mut TcpStream,
-    request: &Timestamped<DaemonRequest>,
-) -> eyre::Result<DaemonReply> {
-    send_message(connection, request)?;
-    if request.inner.expects_tcp_bincode_reply() {
-        receive_reply(connection, Serializer::Bincode)
-            .and_then(|reply| reply.ok_or_else(|| eyre!("server disconnected unexpectedly")))
-    // Use serde json for message with variable length
-    } else if request.inner.expects_tcp_json_reply() {
-        receive_reply(connection, Serializer::SerdeJson)
-            .and_then(|reply| reply.ok_or_else(|| eyre!("server disconnected unexpectedly")))
-    } else {
-        Ok(DaemonReply::Empty)
-    }
-}
-
-fn send_message(
-    connection: &mut TcpStream,
-    message: &Timestamped<DaemonRequest>,
-) -> eyre::Result<()> {
-    let serialized = bincode::serialize(&message).wrap_err("failed to serialize DaemonRequest")?;
-    tcp_send(connection, &serialized).wrap_err("failed to send DaemonRequest")?;
-    Ok(())
-}
-
-fn receive_reply(
-    connection: &mut TcpStream,
-    serializer: Serializer,
-) -> eyre::Result<Option<DaemonReply>> {
-    let raw =
-        match tcp_receive(connection) {
-            Ok(raw) => raw,
-            Err(err) => match err.kind() {
-                std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
-                    return Ok(None);
-                }
-                other => return Err(err).with_context(|| {
-                    format!(
-                        "unexpected I/O error (kind {other:?}) while trying to receive DaemonReply"
-                    )
-                }),
-            },
-        };
-    match serializer {
-        Serializer::Bincode => bincode::deserialize(&raw)
-            .wrap_err("failed to deserialize DaemonReply")
-            .map(Some),
-        Serializer::SerdeJson => serde_json::from_slice(&raw)
-            .wrap_err("failed to deserialize DaemonReply")
-            .map(Some),
-    }
-}
-
-fn tcp_send(connection: &mut (impl Write + Unpin), message: &[u8]) -> std::io::Result<()> {
-    let len_raw = (message.len() as u64).to_le_bytes();
-    connection.write_all(&len_raw)?;
-    connection.write_all(message)?;
-    connection.flush()?;
-    Ok(())
-}
-
-fn tcp_receive(connection: &mut (impl Read + Unpin)) -> std::io::Result<Vec<u8>> {
-    let reply_len = {
-        let mut raw = [0; 8];
-        connection.read_exact(&mut raw)?;
-        u64::from_le_bytes(raw) as usize
-    };
-    let mut reply = vec![0; reply_len];
-    connection.read_exact(&mut reply)?;
-    Ok(reply)
-}

apis/rust/node/src/daemon_connection/unix_domain.rs

@@ -1,86 +0,0 @@
-use dora_message::{
-    daemon_to_node::DaemonReply,
-    node_to_daemon::{DaemonRequest, Timestamped},
-};
-use eyre::{Context, eyre};
-use std::{
-    io::{Read, Write},
-    os::unix::net::UnixStream,
-};
-
-enum Serializer {
-    Bincode,
-    SerdeJson,
-}
-pub fn request(
-    connection: &mut UnixStream,
-    request: &Timestamped<DaemonRequest>,
-) -> eyre::Result<DaemonReply> {
-    send_message(connection, request)?;
-    if request.inner.expects_tcp_bincode_reply() {
-        receive_reply(connection, Serializer::Bincode)
-            .and_then(|reply| reply.ok_or_else(|| eyre!("server disconnected unexpectedly")))
-    // Use serde json for message with variable length
-    } else if request.inner.expects_tcp_json_reply() {
-        receive_reply(connection, Serializer::SerdeJson)
-            .and_then(|reply| reply.ok_or_else(|| eyre!("server disconnected unexpectedly")))
-    } else {
-        Ok(DaemonReply::Empty)
-    }
-}
-
-fn send_message(
-    connection: &mut UnixStream,
-    message: &Timestamped<DaemonRequest>,
-) -> eyre::Result<()> {
-    let serialized = bincode::serialize(&message).wrap_err("failed to serialize DaemonRequest")?;
-    stream_send(connection, &serialized).wrap_err("failed to send DaemonRequest")?;
-    Ok(())
-}
-
-fn receive_reply(
-    connection: &mut UnixStream,
-    serializer: Serializer,
-) -> eyre::Result<Option<DaemonReply>> {
-    let raw =
-        match stream_receive(connection) {
-            Ok(raw) => raw,
-            Err(err) => match err.kind() {
-                std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
-                    return Ok(None);
-                }
-                other => return Err(err).with_context(|| {
-                    format!(
-                        "unexpected I/O error (kind {other:?}) while trying to receive DaemonReply"
-                    )
-                }),
-            },
-        };
-    match serializer {
-        Serializer::Bincode => bincode::deserialize(&raw)
-            .wrap_err("failed to deserialize DaemonReply")
-            .map(Some),
-        Serializer::SerdeJson => serde_json::from_slice(&raw)
-            .wrap_err("failed to deserialize DaemonReply")
-            .map(Some),
-    }
-}
-
-fn stream_send(connection: &mut (impl Write + Unpin), message: &[u8]) -> std::io::Result<()> {
-    let len_raw = (message.len() as u64).to_le_bytes();
-    connection.write_all(&len_raw)?;
-    connection.write_all(message)?;
-    connection.flush()?;
-    Ok(())
-}
-
-fn stream_receive(connection: &mut (impl Read + Unpin)) -> std::io::Result<Vec<u8>> {
-    let reply_len = {
-        let mut raw = [0; 8];
-        connection.read_exact(&mut raw)?;
-        u64::from_le_bytes(raw) as usize
-    };
-    let mut reply = vec![0; reply_len];
-    connection.read_exact(&mut reply)?;
-    Ok(reply)
-}

apis/rust/node/src/event_stream/data_conversion.rs

@@ -1,80 +0,0 @@
-use std::{ptr::NonNull, sync::Arc};
-
-use aligned_vec::{AVec, ConstAlign};
-use dora_arrow_convert::IntoArrow;
-use dora_message::metadata::ArrowTypeInfo;
-use eyre::Context;
-use shared_memory_server::{Shmem, ShmemConf};
-
-use crate::arrow_utils::buffer_into_arrow_array;
-
-pub enum RawData {
-    Empty,
-    Vec(AVec<u8, ConstAlign<128>>),
-    SharedMemory(SharedMemoryData),
-}
-
-impl RawData {
-    pub fn into_arrow_array(
-        self,
-        type_info: &ArrowTypeInfo,
-    ) -> eyre::Result<arrow::array::ArrayData> {
-        let raw_buffer = match self {
-            RawData::Empty => return Ok(().into_arrow().into()),
-            RawData::Vec(data) => {
-                let ptr = NonNull::new(data.as_ptr() as *mut _).unwrap();
-                let len = data.len();
-
-                unsafe { arrow::buffer::Buffer::from_custom_allocation(ptr, len, Arc::new(data)) }
-            }
-            RawData::SharedMemory(data) => {
-                let ptr = NonNull::new(data.data.as_ptr() as *mut _).unwrap();
-                let len = data.data.len();
-
-                unsafe { arrow::buffer::Buffer::from_custom_allocation(ptr, len, Arc::new(data)) }
-            }
-        };
-
-        buffer_into_arrow_array(&raw_buffer, type_info)
-    }
-}
-
-impl std::fmt::Debug for RawData {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Data").finish_non_exhaustive()
-    }
-}
-
-pub struct SharedMemoryData {
-    pub data: MappedInputData,
-    pub _drop: flume::Sender<()>,
-}
-
-pub struct MappedInputData {
-    memory: Box<Shmem>,
-    len: usize,
-}
-
-impl MappedInputData {
-    pub(crate) unsafe fn map(shared_memory_id: &str, len: usize) -> eyre::Result<Self> {
-        let memory = Box::new(
-            ShmemConf::new()
-                .os_id(shared_memory_id)
-                .writable(false)
-                .open()
-                .wrap_err("failed to map shared memory input")?,
-        );
-        Ok(MappedInputData { memory, len })
-    }
-}
-
-impl std::ops::Deref for MappedInputData {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        unsafe { &self.memory.as_slice()[..self.len] }
-    }
-}
-
-unsafe impl Send for MappedInputData {}
-unsafe impl Sync for MappedInputData {}

apis/rust/node/src/event_stream/event.rs

@@ -1,91 +0,0 @@
-use dora_arrow_convert::ArrowData;
-use dora_core::config::{DataId, OperatorId};
-use dora_message::metadata::Metadata;
-
-/// Represents an incoming Dora event.
-///
-/// Events might be triggered by other nodes, by Dora itself, or by some external user input.
-///
-/// It's safe to ignore event types that are not relevant to the node.
-///
-/// This enum is marked as `non_exhaustive` because we might add additional
-/// variants in the future. Please ignore unknown event types instead of throwing an
-/// error to avoid breakage when updating Dora.
-#[derive(Debug)]
-#[non_exhaustive]
-#[allow(clippy::large_enum_variant)]
-pub enum Event {
-    /// An input was received from another node.
-    ///
-    /// This event corresponds to one of the `inputs` of the node as specified
-    /// in the dataflow YAML file.
-    Input {
-        /// The input ID, as specified in the YAML file.
-        ///
-        /// Note that this is not the output ID of the sender, but the ID
-        /// assigned to the input in the YAML file.
-        id: DataId,
-        /// Meta information about this input, e.g. the timestamp.
-        metadata: Metadata,
-        /// The actual data in the Apache Arrow data format.
-        data: ArrowData,
-    },
-    /// An input was closed by the sender.
-    ///
-    /// The sending node mapped to an input exited, so this input will receive
-    /// no more data.
-    InputClosed {
-        /// The ID of the input that was closed, as specified in the YAML file.
-        ///
-        /// Note that this is not the output ID of the sender, but the ID
-        /// assigned to the input in the YAML file.
-        id: DataId,
-    },
-    /// Notification that the event stream is about to close.
-    ///
-    /// The [`StopCause`] field contains the reason for the event stream closure.
-    ///
-    /// Typically, nodes should exit once the event stream closes. One notable
-    /// exception are nodes with no inputs, which will receive aa
-    /// `Event::Stop(StopCause::AllInputsClosed)` right at startup. Source nodes
-    /// might want to keep producing outputs still. (There is currently an open
-    /// discussion of changing this behavior and not sending `AllInputsClosed`
-    /// to nodes without inputs.)
-    ///
-    /// Note: Stop events with `StopCause::Manual` indicate a manual stop operation
-    /// issued through `dora stop` or a `ctrl-c`. Nodes **must exit** once receiving
-    /// such a stop event, otherwise they will be killed by Dora.
-    Stop(StopCause),
-    /// Instructs the node to reload itself or one of its operators.
-    ///
-    /// This event is currently only used for reloading Python operators that are
-    /// started by a `dora runtime` process. So this event should not be sent to normal
-    /// nodes yet.
-    Reload {
-        /// The ID of the operator that should be reloaded.
-        ///
-        /// There is currently no case where `operator_id` is `None`.
-        operator_id: Option<OperatorId>,
-    },
-    /// Notifies the node about an unexpected error that happened inside Dora.
-    ///
-    /// It's a good idea to output or log this error for debugging.
-    Error(String),
-}
-
-/// The reason for closing the event stream.
-///
-/// This enum is marked as `non_exhaustive` because we might add additional
-/// variants in the future.
-#[derive(Debug, Clone)]
-#[non_exhaustive]
-pub enum StopCause {
-    /// The dataflow is stopped early after a `dora stop` command (or on `ctrl-c`).
-    ///
-    /// Nodes should exit as soon as possible if they receive a stop event of
-    /// this type. Dora will kill nodes that keep running for too long after
-    /// receiving such a stop event.
-    Manual,
-    /// The event stream is closed because all of the node's inputs were closed.
-    AllInputsClosed,
-}

apis/rust/node/src/event_stream/merged.rs

@@ -1,144 +0,0 @@
-//! Merge external stream into an [`EventStream`][super::EventStream].
-//!
-//! Sometimes nodes need to listen to external events, in addition to Dora events.
-//! This module provides support for that by providing the [`MergeExternal`] trait.
-
-use futures::{Stream, StreamExt};
-use futures_concurrency::stream::Merge;
-
-/// A Dora event or an event from an external source.
-#[derive(Debug)]
-#[allow(clippy::large_enum_variant)]
-pub enum MergedEvent<E> {
-    /// A Dora event
-    Dora(super::Event),
-    /// An external event
-    ///
-    /// Yielded by the stream that was merged into the Dora [`EventStream`][super::EventStream].
-    External(E),
-}
-
-/// A general enum to represent a value of two possible types.
-pub enum Either<A, B> {
-    /// Value is of the first type, type `A`.
-    First(A),
-    /// Value is of the second type, type `B`.
-    Second(B),
-}
-
-impl<A> Either<A, A> {
-    /// Unwraps an `Either` instance where both types are identical.
-    pub fn flatten(self) -> A {
-        match self {
-            Either::First(a) => a,
-            Either::Second(a) => a,
-        }
-    }
-}
-
-/// Allows merging an external event stream into an existing event stream.
-// TODO: use impl trait return type once stable
-pub trait MergeExternal<'a, E> {
-    /// The item type yielded from the merged stream.
-    type Item;
-
-    /// Merge the given stream into an existing event stream.
-    ///
-    /// Returns a new event stream that yields items from both streams.
-    /// The ordering between the two streams is not guaranteed.
-    fn merge_external(
-        self,
-        external_events: impl Stream<Item = E> + Unpin + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + 'a>;
-}
-
-/// Allows merging a sendable external event stream into an existing (sendable) event stream.
-///
-/// By implementing [`Send`], the streams can be sent to different threads.
-pub trait MergeExternalSend<'a, E> {
-    /// The item type yielded from the merged stream.
-    type Item;
-
-    /// Merge the given stream into an existing event stream.
-    ///
-    /// Returns a new event stream that yields items from both streams.
-    /// The ordering between the two streams is not guaranteed.
-    fn merge_external_send(
-        self,
-        external_events: impl Stream<Item = E> + Unpin + Send + Sync + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + Send + Sync + 'a>;
-}
-
-impl<'a, E> MergeExternal<'a, E> for super::EventStream
-where
-    E: 'static,
-{
-    type Item = MergedEvent<E>;
-
-    fn merge_external(
-        self,
-        external_events: impl Stream<Item = E> + Unpin + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + 'a> {
-        let dora = self.map(MergedEvent::Dora);
-        let external = external_events.map(MergedEvent::External);
-        Box::new((dora, external).merge())
-    }
-}
-
-impl<'a, E> MergeExternalSend<'a, E> for super::EventStream
-where
-    E: 'static,
-{
-    type Item = MergedEvent<E>;
-
-    fn merge_external_send(
-        self,
-        external_events: impl Stream<Item = E> + Unpin + Send + Sync + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + Send + Sync + 'a> {
-        let dora = self.map(MergedEvent::Dora);
-        let external = external_events.map(MergedEvent::External);
-        Box::new((dora, external).merge())
-    }
-}
-
-impl<'a, E, F, S> MergeExternal<'a, F> for S
-where
-    S: Stream<Item = MergedEvent<E>> + Unpin + 'a,
-    E: 'a,
-    F: 'a,
-{
-    type Item = MergedEvent<Either<E, F>>;
-
-    fn merge_external(
-        self,
-        external_events: impl Stream<Item = F> + Unpin + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + 'a> {
-        let first = self.map(|e| match e {
-            MergedEvent::Dora(d) => MergedEvent::Dora(d),
-            MergedEvent::External(e) => MergedEvent::External(Either::First(e)),
-        });
-        let second = external_events.map(|e| MergedEvent::External(Either::Second(e)));
-        Box::new((first, second).merge())
-    }
-}
-
-impl<'a, E, F, S> MergeExternalSend<'a, F> for S
-where
-    S: Stream<Item = MergedEvent<E>> + Unpin + Send + Sync + 'a,
-    E: 'a,
-    F: 'a,
-{
-    type Item = MergedEvent<Either<E, F>>;
-
-    fn merge_external_send(
-        self,
-        external_events: impl Stream<Item = F> + Unpin + Send + Sync + 'a,
-    ) -> Box<dyn Stream<Item = Self::Item> + Unpin + Send + Sync + 'a> {
-        let first = self.map(|e| match e {
-            MergedEvent::Dora(d) => MergedEvent::Dora(d),
-            MergedEvent::External(e) => MergedEvent::External(Either::First(e)),
-        });
-        let second = external_events.map(|e| MergedEvent::External(Either::Second(e)));
-        Box::new((first, second).merge())
-    }
-}

apis/rust/node/src/event_stream/mod.rs

@@ -1,366 +0,0 @@
-use std::{
-    collections::{BTreeMap, HashMap, VecDeque},
-    pin::pin,
-    sync::Arc,
-    time::Duration,
-};
-
-use dora_message::{
-    DataflowId,
-    daemon_to_node::{DaemonCommunication, DaemonReply, DataMessage, NodeEvent},
-    id::DataId,
-    node_to_daemon::{DaemonRequest, Timestamped},
-};
-pub use event::{Event, StopCause};
-use futures::{
-    Stream, StreamExt,
-    future::{Either, select},
-};
-use futures_timer::Delay;
-use scheduler::{NON_INPUT_EVENT, Scheduler};
-
-use self::thread::{EventItem, EventStreamThreadHandle};
-use crate::{
-    daemon_connection::DaemonChannel,
-    event_stream::data_conversion::{MappedInputData, RawData, SharedMemoryData},
-};
-use dora_core::{
-    config::{Input, NodeId},
-    uhlc,
-};
-use eyre::{Context, eyre};
-
-pub use scheduler::Scheduler as EventScheduler;
-
-mod data_conversion;
-mod event;
-pub mod merged;
-mod scheduler;
-mod thread;
-
-/// Asynchronous iterator over the incoming [`Event`]s destined for this node.
-///
-/// This struct [implements](#impl-Stream-for-EventStream) the [`Stream`] trait,
-/// so you can use methods of the [`StreamExt`] trait
-/// on this struct. A common pattern is `while let Some(event) = event_stream.next().await`.
-///
-/// Nodes should iterate over this event stream and react to events that they are interested in.
-/// Typically, the most important event type is [`Event::Input`].
-/// You don't need to handle all events, it's fine to ignore events that are not relevant to your node.
-///
-/// The event stream will close itself after a [`Event::Stop`] was received.
-/// A manual `break` on [`Event::Stop`] is typically not needed.
-/// _(You probably do need to use a manual `break` on stop events when using the
-/// [`StreamExt::merge`][`futures_concurrency::stream::StreamExt::merge`] implementation on
-/// [`EventStream`] to combine the stream with an external one.)_
-///
-/// Once the event stream finished, nodes should exit.
-/// Note that Dora kills nodes that don't exit quickly after a [`Event::Stop`] of type
-/// [`StopCause::Manual`] was received.
-pub struct EventStream {
-    node_id: NodeId,
-    receiver: flume::r#async::RecvStream<'static, EventItem>,
-    _thread_handle: EventStreamThreadHandle,
-    close_channel: DaemonChannel,
-    clock: Arc<uhlc::HLC>,
-    scheduler: Scheduler,
-}
-
-impl EventStream {
-    #[tracing::instrument(level = "trace", skip(clock))]
-    pub(crate) fn init(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        daemon_communication: &DaemonCommunication,
-        input_config: BTreeMap<DataId, Input>,
-        clock: Arc<uhlc::HLC>,
-    ) -> eyre::Result<Self> {
-        let channel = match daemon_communication {
-            DaemonCommunication::Shmem {
-                daemon_events_region_id,
-                ..
-            } => unsafe { DaemonChannel::new_shmem(daemon_events_region_id) }.wrap_err_with(
-                || format!("failed to create shmem event stream for node `{node_id}`"),
-            )?,
-            DaemonCommunication::Tcp { socket_addr } => DaemonChannel::new_tcp(*socket_addr)
-                .wrap_err_with(|| format!("failed to connect event stream for node `{node_id}`"))?,
-            #[cfg(unix)]
-            DaemonCommunication::UnixDomain { socket_file } => {
-                DaemonChannel::new_unix_socket(socket_file).wrap_err_with(|| {
-                    format!("failed to connect event stream for node `{node_id}`")
-                })?
-            }
-        };
-
-        let close_channel = match daemon_communication {
-            DaemonCommunication::Shmem {
-                daemon_events_close_region_id,
-                ..
-            } => unsafe { DaemonChannel::new_shmem(daemon_events_close_region_id) }.wrap_err_with(
-                || format!("failed to create shmem event close channel for node `{node_id}`"),
-            )?,
-            DaemonCommunication::Tcp { socket_addr } => DaemonChannel::new_tcp(*socket_addr)
-                .wrap_err_with(|| {
-                    format!("failed to connect event close channel for node `{node_id}`")
-                })?,
-            #[cfg(unix)]
-            DaemonCommunication::UnixDomain { socket_file } => {
-                DaemonChannel::new_unix_socket(socket_file).wrap_err_with(|| {
-                    format!("failed to connect event close channel for node `{node_id}`")
-                })?
-            }
-        };
-
-        let mut queue_size_limit: HashMap<DataId, (usize, VecDeque<EventItem>)> = input_config
-            .iter()
-            .map(|(input, config)| {
-                (
-                    input.clone(),
-                    (config.queue_size.unwrap_or(1), VecDeque::new()),
-                )
-            })
-            .collect();
-
-        queue_size_limit.insert(
-            DataId::from(NON_INPUT_EVENT.to_string()),
-            (1_000, VecDeque::new()),
-        );
-
-        let scheduler = Scheduler::new(queue_size_limit);
-
-        Self::init_on_channel(
-            dataflow_id,
-            node_id,
-            channel,
-            close_channel,
-            clock,
-            scheduler,
-        )
-    }
-
-    pub(crate) fn init_on_channel(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        mut channel: DaemonChannel,
-        mut close_channel: DaemonChannel,
-        clock: Arc<uhlc::HLC>,
-        scheduler: Scheduler,
-    ) -> eyre::Result<Self> {
-        channel.register(dataflow_id, node_id.clone(), clock.new_timestamp())?;
-        let reply = channel
-            .request(&Timestamped {
-                inner: DaemonRequest::Subscribe,
-                timestamp: clock.new_timestamp(),
-            })
-            .map_err(|e| eyre!(e))
-            .wrap_err("failed to create subscription with dora-daemon")?;
-
-        match reply {
-            DaemonReply::Result(Ok(())) => {}
-            DaemonReply::Result(Err(err)) => {
-                eyre::bail!("subscribe failed: {err}")
-            }
-            other => eyre::bail!("unexpected subscribe reply: {other:?}"),
-        }
-
-        close_channel.register(dataflow_id, node_id.clone(), clock.new_timestamp())?;
-
-        let (tx, rx) = flume::bounded(100_000_000);
-
-        let thread_handle = thread::init(node_id.clone(), tx, channel, clock.clone())?;
-
-        Ok(EventStream {
-            node_id: node_id.clone(),
-            receiver: rx.into_stream(),
-            _thread_handle: thread_handle,
-            close_channel,
-            clock,
-            scheduler,
-        })
-    }
-
-    /// Synchronously waits for the next event.
-    ///
-    /// Blocks the thread until the next event arrives.
-    /// Returns [`None`] once the event stream is closed.
-    ///
-    /// For an asynchronous variant of this method see [`recv_async`][Self::recv_async].
-    ///
-    /// ## Event Reordering
-    ///
-    /// This method uses an [`EventScheduler`] internally to **reorder events**. This means that the
-    /// events might be returned in a different order than they occurred. For details, check the
-    /// documentation of the [`EventScheduler`] struct.
-    ///
-    /// If you want to receive the events in their original chronological order, use the
-    /// asynchronous [`StreamExt::next`] method instead ([`EventStream`] implements the
-    /// [`Stream`] trait).
-    pub fn recv(&mut self) -> Option<Event> {
-        futures::executor::block_on(self.recv_async())
-    }
-
-    /// Receives the next incoming [`Event`] synchronously with a timeout.
-    ///
-    /// Blocks the thread until the next event arrives or the timeout is reached.
-    /// Returns a [`Event::Error`] if no event was received within the given duration.
-    ///
-    /// Returns [`None`] once the event stream is closed.
-    ///
-    /// For an asynchronous variant of this method see [`recv_async_timeout`][Self::recv_async_timeout].
-    ///
-    /// ## Event Reordering
-    ///
-    /// This method uses an [`EventScheduler`] internally to **reorder events**. This means that the
-    /// events might be returned in a different order than they occurred. For details, check the
-    /// documentation of the [`EventScheduler`] struct.
-    ///
-    /// If you want to receive the events in their original chronological order, use the
-    /// asynchronous [`StreamExt::next`] method instead ([`EventStream`] implements the
-    /// [`Stream`] trait).
-    pub fn recv_timeout(&mut self, dur: Duration) -> Option<Event> {
-        futures::executor::block_on(self.recv_async_timeout(dur))
-    }
-
-    /// Receives the next incoming [`Event`] asynchronously, using an [`EventScheduler`] for fairness.
-    ///
-    /// Returns [`None`] once the event stream is closed.
-    ///
-    /// ## Event Reordering
-    ///
-    /// This method uses an [`EventScheduler`] internally to **reorder events**. This means that the
-    /// events might be returned in a different order than they occurred. For details, check the
-    /// documentation of the [`EventScheduler`] struct.
-    ///
-    /// If you want to receive the events in their original chronological order, use the
-    /// [`StreamExt::next`] method with a custom timeout future instead
-    /// ([`EventStream`] implements the [`Stream`] trait).
-    pub async fn recv_async(&mut self) -> Option<Event> {
-        loop {
-            if self.scheduler.is_empty() {
-                if let Some(event) = self.receiver.next().await {
-                    self.scheduler.add_event(event);
-                } else {
-                    break;
-                }
-            } else {
-                match select(Delay::new(Duration::from_micros(300)), self.receiver.next()).await {
-                    Either::Left((_elapsed, _)) => break,
-                    Either::Right((Some(event), _)) => self.scheduler.add_event(event),
-                    Either::Right((None, _)) => break,
-                };
-            }
-        }
-        let event = self.scheduler.next();
-        event.map(Self::convert_event_item)
-    }
-
-    /// Receives the next incoming [`Event`] asynchronously with a timeout.
-    ///
-    /// Returns a [`Event::Error`] if no event was received within the given duration.
-    ///
-    /// Returns [`None`] once the event stream is closed.
-    ///
-    /// ## Event Reordering
-    ///
-    /// This method uses an [`EventScheduler`] internally to **reorder events**. This means that the
-    /// events might be returned in a different order than they occurred. For details, check the
-    /// documentation of the [`EventScheduler`] struct.
-    ///
-    /// If you want to receive the events in their original chronological order, use the
-    /// [`StreamExt::next`] method with a custom timeout future instead
-    /// ([`EventStream`] implements the [`Stream`] trait).
-    pub async fn recv_async_timeout(&mut self, dur: Duration) -> Option<Event> {
-        match select(Delay::new(dur), pin!(self.recv_async())).await {
-            Either::Left((_elapsed, _)) => Some(Self::convert_event_item(EventItem::TimeoutError(
-                eyre!("Receiver timed out"),
-            ))),
-            Either::Right((event, _)) => event,
-        }
-    }
-
-    fn convert_event_item(item: EventItem) -> Event {
-        match item {
-            EventItem::NodeEvent { event, ack_channel } => match event {
-                NodeEvent::Stop => Event::Stop(event::StopCause::Manual),
-                NodeEvent::Reload { operator_id } => Event::Reload { operator_id },
-                NodeEvent::InputClosed { id } => Event::InputClosed { id },
-                NodeEvent::Input { id, metadata, data } => {
-                    let data = match data {
-                        None => Ok(None),
-                        Some(DataMessage::Vec(v)) => Ok(Some(RawData::Vec(v))),
-                        Some(DataMessage::SharedMemory {
-                            shared_memory_id,
-                            len,
-                            drop_token: _, // handled in `event_stream_loop`
-                        }) => unsafe {
-                            MappedInputData::map(&shared_memory_id, len).map(|data| {
-                                Some(RawData::SharedMemory(SharedMemoryData {
-                                    data,
-                                    _drop: ack_channel,
-                                }))
-                            })
-                        },
-                    };
-                    let data = data.and_then(|data| {
-                        let raw_data = data.unwrap_or(RawData::Empty);
-                        raw_data
-                            .into_arrow_array(&metadata.type_info)
-                            .map(arrow::array::make_array)
-                    });
-                    match data {
-                        Ok(data) => Event::Input {
-                            id,
-                            metadata,
-                            data: data.into(),
-                        },
-                        Err(err) => Event::Error(format!("{err:?}")),
-                    }
-                }
-                NodeEvent::AllInputsClosed => Event::Stop(event::StopCause::AllInputsClosed),
-            },
-
-            EventItem::FatalError(err) => {
-                Event::Error(format!("fatal event stream error: {err:?}"))
-            }
-            EventItem::TimeoutError(err) => {
-                Event::Error(format!("Timeout event stream error: {err:?}"))
-            }
-        }
-    }
-}
-
-impl Stream for EventStream {
-    type Item = Event;
-
-    fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        self.receiver
-            .poll_next_unpin(cx)
-            .map(|item| item.map(Self::convert_event_item))
-    }
-}
-
-impl Drop for EventStream {
-    #[tracing::instrument(skip(self), fields(%self.node_id))]
-    fn drop(&mut self) {
-        let request = Timestamped {
-            inner: DaemonRequest::EventStreamDropped,
-            timestamp: self.clock.new_timestamp(),
-        };
-        let result = self
-            .close_channel
-            .request(&request)
-            .map_err(|e| eyre!(e))
-            .wrap_err("failed to signal event stream closure to dora-daemon")
-            .and_then(|r| match r {
-                DaemonReply::Result(Ok(())) => Ok(()),
-                DaemonReply::Result(Err(err)) => Err(eyre!("EventStreamClosed failed: {err}")),
-                other => Err(eyre!("unexpected EventStreamClosed reply: {other:?}")),
-            });
-        if let Err(err) = result {
-            tracing::warn!("{err:?}")
-        }
-    }
-}

apis/rust/node/src/event_stream/scheduler.rs

@@ -1,159 +0,0 @@
-use std::collections::{HashMap, VecDeque};
-
-use dora_message::{daemon_to_node::NodeEvent, id::DataId};
-
-use super::thread::EventItem;
-pub(crate) const NON_INPUT_EVENT: &str = "dora/non_input_event";
-
-/// This scheduler will make sure that there is fairness between inputs.
-///
-/// The scheduler reorders events in the following way:
-///
-/// - **Non-input events are prioritized**
-///   
-///   If the node received any events that are not input events, they are returned first. The
-///   intention of this reordering is that the nodes can react quickly to dataflow-related events
-///   even when their input queues are very full.
-///   
-///   This reordering has some side effects that might be unexpected:
-///   - An [`InputClosed`][super::Event::InputClosed] event might be yielded before the last
-///     input events of that ID.
-///     
-///     Usually, an `InputClosed` event indicates that there won't be any subsequent inputs
-///     of a certain ID. This invariant does not hold anymore for a scheduled event stream.
-///   - The [`Stop`][super::Event::Stop] event might not be the last event of the stream anymore.
-///     
-///     Usually, the `Stop` event is the last event that is sent to a node before the event stream
-///     is closed. Because of the reordering, the stream might return more events after a `Stop`
-///     event.
-/// - **Input events are grouped by ID** and yielded in a **least-recently used order (by ID)**.
-///
-///   The scheduler keeps a separate queue for each input ID, where the incoming input events are
-///   placed in their chronological order. When yielding the next event, the scheduler iterates over
-///   these queues in least-recently used order. This means that the queue corresponding to the
-///   last yielded event will be checked last. The scheduler will return the oldest event from the
-///   first non-empty queue.
-///
-///   The side effect of this change is that inputs events of different IDs are no longer in their
-///   chronological order. This might lead to unexpected results for input events that are caused by
-///   each other.
-///
-/// ## Example 1
-/// Consider the case that one input has a very high frequency and another one with a very slow
-/// frequency. The event stream will always alternate between the two inputs when each input is
-/// available.
-/// Without the scheduling, the high-frequency input would be returned much more often.
-///
-/// ## Example 2
-/// Again, let's consider the case that one input has a very high frequency and the other has a
-/// very slow frequency. This time, we define a small maximum queue sizes for the low-frequency
-/// input, but a large queue size for the high-frequency one.
-/// Using the scheduler, the event stream will always alternate between high and low-frequency
-/// inputs as long as inputs of both types are available.
-///
-/// Without scheduling, the low-frequency input might never be yielded before
-/// it's dropped because there is almost always an older high-frequency input available that is
-/// yielded first. Once the low-frequency input would be the next one chronologically, it might
-/// have been dropped already because the node received newer low-frequency inputs in the
-/// meantime (the queue length is small). At this point, the next-oldest input is a high-frequency
-/// input again.
-///
-/// ## Example 3
-/// Consider a high-frequency camera input and a low-frequency bounding box input, which is based
-/// on the latest camera image. The dataflow YAML file specifies a large queue size for the camera
-/// input and a small queue size for the bounding box input.
-///
-/// With scheduling, the number of
-/// buffered camera inputs might grow over time. As a result the camera inputs yielded from the
-/// stream (in oldest-first order) are not synchronized with the bounding box inputs anymore. So
-/// the node receives an up-to-date bounding box, but a considerably outdated image.
-///
-/// Without scheduling, the events are returned in chronological order. This time, the bounding
-/// box might be slightly outdated if the camera sent new images before the bounding box was
-/// ready. However, the time difference between the two input types is independent of the
-/// queue size this time.
-///
-/// (If a perfect matching bounding box is required, we recommend to forward the input image as
-/// part of the bounding box output. This way, the receiving node only needs to subscribe to one
-/// input so no mismatches can happen.)
-#[derive(Debug)]
-pub struct Scheduler {
-    /// Tracks the last-used event ID
-    last_used: VecDeque<DataId>,
-    /// Tracks events per ID
-    event_queues: HashMap<DataId, (usize, VecDeque<EventItem>)>,
-}
-
-impl Scheduler {
-    pub(crate) fn new(event_queues: HashMap<DataId, (usize, VecDeque<EventItem>)>) -> Self {
-        let topic = VecDeque::from_iter(
-            event_queues
-                .keys()
-                .filter(|t| **t != DataId::from(NON_INPUT_EVENT.to_string()))
-                .cloned(),
-        );
-        Self {
-            last_used: topic,
-            event_queues,
-        }
-    }
-
-    pub(crate) fn add_event(&mut self, event: EventItem) {
-        let event_id = match &event {
-            EventItem::NodeEvent {
-                event:
-                    NodeEvent::Input {
-                        id,
-                        metadata: _,
-                        data: _,
-                    },
-                ack_channel: _,
-            } => id,
-            _ => &DataId::from(NON_INPUT_EVENT.to_string()),
-        };
-
-        // Enforce queue size limit
-        if let Some((size, queue)) = self.event_queues.get_mut(event_id) {
-            // Remove the oldest event if at limit
-            if &queue.len() >= size {
-                tracing::debug!("Discarding event for input `{event_id}` due to queue size limit");
-                queue.pop_front();
-            }
-            queue.push_back(event);
-        } else {
-            unimplemented!("Received an event that was not in the definition event id description.")
-        }
-    }
-
-    pub(crate) fn next(&mut self) -> Option<EventItem> {
-        // Retrieve message from the non input event first that have priority over input message.
-        if let Some((_size, queue)) = self
-            .event_queues
-            .get_mut(&DataId::from(NON_INPUT_EVENT.to_string()))
-        {
-            if let Some(event) = queue.pop_front() {
-                return Some(event);
-            }
-        }
-
-        // Process the ID with the oldest timestamp using BTreeMap Ordering
-        for (index, id) in self.last_used.clone().iter().enumerate() {
-            if let Some((_size, queue)) = self.event_queues.get_mut(id) {
-                if let Some(event) = queue.pop_front() {
-                    // Put last used at last
-                    self.last_used.remove(index);
-                    self.last_used.push_back(id.clone());
-                    return Some(event);
-                }
-            }
-        }
-
-        None
-    }
-
-    pub(crate) fn is_empty(&self) -> bool {
-        self.event_queues
-            .iter()
-            .all(|(_id, (_size, queue))| queue.is_empty())
-    }
-}

apis/rust/node/src/event_stream/thread.rs

@@ -1,283 +0,0 @@
-use dora_core::{
-    config::NodeId,
-    uhlc::{self, Timestamp},
-};
-use dora_message::{
-    daemon_to_node::{DaemonReply, NodeEvent},
-    node_to_daemon::{DaemonRequest, DropToken, Timestamped},
-};
-use eyre::{Context, eyre};
-use flume::RecvTimeoutError;
-use std::{
-    sync::Arc,
-    time::{Duration, Instant},
-};
-
-use crate::daemon_connection::DaemonChannel;
-
-pub fn init(
-    node_id: NodeId,
-    tx: flume::Sender<EventItem>,
-    channel: DaemonChannel,
-    clock: Arc<uhlc::HLC>,
-) -> eyre::Result<EventStreamThreadHandle> {
-    let node_id_cloned = node_id.clone();
-    let join_handle = std::thread::spawn(|| event_stream_loop(node_id_cloned, tx, channel, clock));
-    Ok(EventStreamThreadHandle::new(node_id, join_handle))
-}
-
-#[derive(Debug)]
-#[allow(clippy::large_enum_variant)]
-pub enum EventItem {
-    NodeEvent {
-        event: NodeEvent,
-        ack_channel: flume::Sender<()>,
-    },
-    FatalError(eyre::Report),
-    TimeoutError(eyre::Report),
-}
-
-pub struct EventStreamThreadHandle {
-    node_id: NodeId,
-    handle: flume::Receiver<std::thread::Result<()>>,
-}
-
-impl EventStreamThreadHandle {
-    fn new(node_id: NodeId, join_handle: std::thread::JoinHandle<()>) -> Self {
-        let (tx, rx) = flume::bounded(1);
-        std::thread::spawn(move || {
-            let _ = tx.send(join_handle.join());
-        });
-        Self {
-            node_id,
-            handle: rx,
-        }
-    }
-}
-
-impl Drop for EventStreamThreadHandle {
-    #[tracing::instrument(skip(self), fields(node_id = %self.node_id))]
-    fn drop(&mut self) {
-        if self.handle.is_empty() {
-            tracing::trace!("waiting for event stream thread");
-        }
-
-        // TODO: The event stream duration has been shorten due to
-        // Python Reference Counting not working properly and deleting the node
-        // before deleting event creating a race condition.
-        //
-        // In the future, we hope to fix this issue so that
-        // the event stream can be properly waited for every time.
-        match self.handle.recv_timeout(Duration::from_secs(1)) {
-            Ok(Ok(())) => {
-                tracing::trace!("event stream thread finished");
-            }
-            Ok(Err(_)) => {
-                tracing::error!("event stream thread panicked");
-            }
-            Err(RecvTimeoutError::Timeout) => {
-                tracing::warn!("timeout while waiting for event stream thread");
-            }
-            Err(RecvTimeoutError::Disconnected) => {
-                tracing::warn!("event stream thread result channel closed unexpectedly");
-            }
-        }
-    }
-}
-
-#[tracing::instrument(skip(tx, channel, clock))]
-fn event_stream_loop(
-    node_id: NodeId,
-    tx: flume::Sender<EventItem>,
-    mut channel: DaemonChannel,
-    clock: Arc<uhlc::HLC>,
-) {
-    let mut tx = Some(tx);
-    let mut close_tx = false;
-    let mut pending_drop_tokens: Vec<(DropToken, flume::Receiver<()>, Instant, u64)> = Vec::new();
-    let mut drop_tokens = Vec::new();
-
-    let result = 'outer: loop {
-        if let Err(err) = handle_pending_drop_tokens(&mut pending_drop_tokens, &mut drop_tokens) {
-            break 'outer Err(err);
-        }
-
-        let daemon_request = Timestamped {
-            inner: DaemonRequest::NextEvent {
-                drop_tokens: std::mem::take(&mut drop_tokens),
-            },
-            timestamp: clock.new_timestamp(),
-        };
-        let events = match channel.request(&daemon_request) {
-            Ok(DaemonReply::NextEvents(events)) => {
-                if events.is_empty() {
-                    tracing::trace!("event stream closed for node `{node_id}`");
-                    break Ok(());
-                } else {
-                    events
-                }
-            }
-            Ok(other) => {
-                let err = eyre!("unexpected control reply: {other:?}");
-                tracing::warn!("{err:?}");
-                continue;
-            }
-            Err(err) => {
-                let err = eyre!(err).wrap_err("failed to receive incoming event");
-                tracing::warn!("{err:?}");
-                continue;
-            }
-        };
-        for Timestamped { inner, timestamp } in events {
-            if let Err(err) = clock.update_with_timestamp(&timestamp) {
-                tracing::warn!("failed to update HLC: {err}");
-            }
-            let drop_token = match &inner {
-                NodeEvent::Input {
-                    data: Some(data), ..
-                } => data.drop_token(),
-                NodeEvent::AllInputsClosed => {
-                    close_tx = true;
-                    None
-                }
-                _ => None,
-            };
-
-            if let Some(tx) = tx.as_ref() {
-                let (drop_tx, drop_rx) = flume::bounded(0);
-                match tx.send(EventItem::NodeEvent {
-                    event: inner,
-                    ack_channel: drop_tx,
-                }) {
-                    Ok(()) => {}
-                    Err(send_error) => {
-                        let event = send_error.into_inner();
-                        tracing::trace!(
-                            "event channel was closed already, could not forward `{event:?}`"
-                        );
-
-                        break 'outer Ok(());
-                    }
-                }
-
-                if let Some(token) = drop_token {
-                    pending_drop_tokens.push((token, drop_rx, Instant::now(), 1));
-                }
-            } else {
-                tracing::warn!("dropping event because event `tx` was already closed: `{inner:?}`");
-            }
-
-            if close_tx {
-                tx = None;
-            };
-        }
-    };
-    if let Err(err) = result {
-        if let Some(tx) = tx.as_ref() {
-            if let Err(flume::SendError(item)) = tx.send(EventItem::FatalError(err)) {
-                let err = match item {
-                    EventItem::FatalError(err) => err,
-                    _ => unreachable!(),
-                };
-                tracing::error!("failed to report fatal EventStream error: {err:?}");
-            }
-        } else {
-            tracing::error!("received error event after `tx` was closed: {err:?}");
-        }
-    }
-
-    if let Err(err) = report_remaining_drop_tokens(
-        channel,
-        drop_tokens,
-        pending_drop_tokens,
-        clock.new_timestamp(),
-    )
-    .context("failed to report remaining drop tokens")
-    {
-        tracing::warn!("{err:?}");
-    }
-}
-
-fn handle_pending_drop_tokens(
-    pending_drop_tokens: &mut Vec<(DropToken, flume::Receiver<()>, Instant, u64)>,
-    drop_tokens: &mut Vec<DropToken>,
-) -> eyre::Result<()> {
-    let mut still_pending = Vec::new();
-    for (token, rx, since, warn) in pending_drop_tokens.drain(..) {
-        match rx.try_recv() {
-            Ok(()) => return Err(eyre!("Node API should not send anything on ACK channel")),
-            Err(flume::TryRecvError::Disconnected) => {
-                // the event was dropped -> add the drop token to the list
-                drop_tokens.push(token);
-            }
-            Err(flume::TryRecvError::Empty) => {
-                let duration = Duration::from_secs(30 * warn);
-                if since.elapsed() > duration {
-                    tracing::warn!("timeout: token {token:?} was not dropped after {duration:?}");
-                }
-                still_pending.push((token, rx, since, warn + 1));
-            }
-        }
-    }
-    *pending_drop_tokens = still_pending;
-    Ok(())
-}
-
-fn report_remaining_drop_tokens(
-    mut channel: DaemonChannel,
-    mut drop_tokens: Vec<DropToken>,
-    mut pending_drop_tokens: Vec<(DropToken, flume::Receiver<()>, Instant, u64)>,
-    timestamp: Timestamp,
-) -> eyre::Result<()> {
-    while !(pending_drop_tokens.is_empty() && drop_tokens.is_empty()) {
-        report_drop_tokens(&mut drop_tokens, &mut channel, timestamp)?;
-
-        let mut still_pending = Vec::new();
-        for (token, rx, since, _) in pending_drop_tokens.drain(..) {
-            match rx.recv_timeout(Duration::from_millis(100)) {
-                Ok(()) => return Err(eyre!("Node API should not send anything on ACK channel")),
-                Err(flume::RecvTimeoutError::Disconnected) => {
-                    // the event was dropped -> add the drop token to the list
-                    drop_tokens.push(token);
-                }
-                Err(flume::RecvTimeoutError::Timeout) => {
-                    let duration = Duration::from_secs(1);
-                    if since.elapsed() > duration {
-                        tracing::warn!(
-                            "timeout: node finished, but token {token:?} was still not \
-                            dropped after {duration:?} -> ignoring it"
-                        );
-                    } else {
-                        still_pending.push((token, rx, since, 0));
-                    }
-                }
-            }
-        }
-        pending_drop_tokens = still_pending;
-        if !pending_drop_tokens.is_empty() {
-            tracing::trace!("waiting for drop for {} events", pending_drop_tokens.len());
-        }
-    }
-
-    Ok(())
-}
-
-fn report_drop_tokens(
-    drop_tokens: &mut Vec<DropToken>,
-    channel: &mut DaemonChannel,
-    timestamp: Timestamp,
-) -> Result<(), eyre::ErrReport> {
-    if drop_tokens.is_empty() {
-        return Ok(());
-    }
-    let daemon_request = Timestamped {
-        inner: DaemonRequest::ReportDropTokens {
-            drop_tokens: std::mem::take(drop_tokens),
-        },
-        timestamp,
-    };
-    match channel.request(&daemon_request)? {
-        DaemonReply::Empty => Ok(()),
-        other => Err(eyre!("unexpected ReportDropTokens reply: {other:?}")),
-    }
-}

apis/rust/node/src/lib.rs

@@ -1,97 +0,0 @@
-//! This crate enables you to create nodes for the [Dora] dataflow framework.
-//!
-//! [Dora]: https://dora-rs.ai/
-//!
-//! ## The Dora Framework
-//!
-//! Dora is a dataflow frame work that models applications as a directed graph, with nodes
-//! representing operations and edges representing data transfer.
-//! The layout of the dataflow graph is defined through a YAML file in Dora.
-//! For details, see our [Dataflow Specification](https://dora-rs.ai/docs/api/dataflow-config/)
-//! chapter.
-//!
-//! Dora nodes are typically spawned by the Dora framework, instead of spawning them manually.
-//! If you want to spawn a node manually, define it as a [_dynamic_ node](#dynamic-nodes).
-//!
-//! ## Normal Usage
-//!
-//! In order to connect your executable to Dora, you need to initialize a [`DoraNode`].
-//! For standard nodes, the recommended initialization function is [`init_from_env`][`DoraNode::init_from_env`].
-//! This function will return two values, a [`DoraNode`] instance and an [`EventStream`]:
-//!
-//! ```no_run
-//! use dora_node_api::DoraNode;
-//!
-//! let (mut node, mut events) = DoraNode::init_from_env()?;
-//! # Ok::<(), eyre::Report>(())
-//! ```
-//!
-//! You can use the `node` instance to send outputs and retrieve information about the node and
-//! the dataflow. The `events` stream yields the inputs that the node defines in the dataflow
-//! YAML file and other incoming events.
-//!
-//! ### Sending Outputs
-//!
-//! The [`DoraNode`] instance enables you to send outputs in different formats.
-//! For best performance, use the [Arrow](https://arrow.apache.org/docs/index.html) data format
-//! and one of the output functions that utilizes shared memory.
-//!
-//! ### Receiving Events
-//!
-//! The [`EventStream`] is an [`AsyncIterator`][std::async_iter::AsyncIterator] that yields the incoming [`Event`]s.
-//!
-//! Nodes should iterate over this event stream and react to events that they are interested in.
-//! Typically, the most important event type is [`Event::Input`].
-//! You don't need to handle all events, it's fine to ignore events that are not relevant to your node.
-//!
-//! The event stream will close itself after a [`Event::Stop`] was received.
-//! A manual `break` on [`Event::Stop`] is typically not needed.
-//! _(You probably do need to use a manual `break` on stop events when using the
-//! [`StreamExt::merge`][`futures_concurrency::stream::StreamExt::merge`] implementation on
-//! [`EventStream`] to combine the stream with an external one.)_
-//!
-//! Once the event stream finished, nodes should exit.
-//! Note that Dora kills nodes that don't exit quickly after a [`Event::Stop`] of type
-//! [`StopCause::Manual`] was received.
-//!
-//!
-//!
-//! ## Dynamic Nodes
-//!
-//! <div class="warning">
-//!
-//! Dynamic nodes have certain [limitations](#limitations). Use with care.
-//!
-//! </div>
-//!
-//! Nodes can be defined as `dynamic` by setting their `path` attribute to `dynamic` in the
-//! dataflow YAML file. Dynamic nodes are not spawned by the Dora framework and need to be started
-//! manually.
-//!
-//! Dynamic nodes cannot use the [`DoraNode::init_from_env`] function for initialization.
-//! Instead, they can be initialized through the [`DoraNode::init_from_node_id`] function.
-//!
-//! ### Limitations
-//!
-//! - Dynamic nodes **don't work with `dora run`**.
-//! - As dynamic nodes are identified by their node ID, this **ID must be unique**
-//!   across all running dataflows.
-//! - For distributed dataflows, nodes need to be manually spawned on the correct machine.
-
-#![warn(missing_docs)]
-
-pub use arrow;
-pub use dora_arrow_convert::*;
-pub use dora_core::{self, uhlc};
-pub use dora_message::{
-    DataflowId,
-    metadata::{Metadata, MetadataParameters, Parameter},
-};
-pub use event_stream::{Event, EventScheduler, EventStream, StopCause, merged};
-pub use flume::Receiver;
-pub use futures;
-pub use node::{DataSample, DoraNode, ZERO_COPY_THRESHOLD, arrow_utils};
-
-mod daemon_connection;
-mod event_stream;
-mod node;

apis/rust/node/src/node/arrow_utils.rs

@@ -1,117 +0,0 @@
-//! Utility functions for converting Arrow arrays to/from raw data.
-//!
-use arrow::array::{ArrayData, BufferSpec};
-use dora_message::metadata::{ArrowTypeInfo, BufferOffset};
-use eyre::Context;
-
-/// Calculates the data size in bytes required for storing a continuous copy of the given Arrow
-/// array.
-pub fn required_data_size(array: &ArrayData) -> usize {
-    let mut next_offset = 0;
-    required_data_size_inner(array, &mut next_offset);
-    next_offset
-}
-fn required_data_size_inner(array: &ArrayData, next_offset: &mut usize) {
-    let layout = arrow::array::layout(array.data_type());
-    for (buffer, spec) in array.buffers().iter().zip(&layout.buffers) {
-        // consider alignment padding
-        if let BufferSpec::FixedWidth { alignment, .. } = *spec {
-            *next_offset = (*next_offset).div_ceil(alignment) * alignment;
-        }
-        *next_offset += buffer.len();
-    }
-    for child in array.child_data() {
-        required_data_size_inner(child, next_offset);
-    }
-}
-
-/// Copy the given Arrow array into the provided buffer.
-///
-/// If the Arrow array consists of multiple buffers, they are placed continuously in the target
-/// buffer (there might be some padding for alignment)
-///
-/// Panics if the buffer is not large enough.
-pub fn copy_array_into_sample(target_buffer: &mut [u8], arrow_array: &ArrayData) -> ArrowTypeInfo {
-    let mut next_offset = 0;
-    copy_array_into_sample_inner(target_buffer, &mut next_offset, arrow_array)
-}
-
-fn copy_array_into_sample_inner(
-    target_buffer: &mut [u8],
-    next_offset: &mut usize,
-    arrow_array: &ArrayData,
-) -> ArrowTypeInfo {
-    let mut buffer_offsets = Vec::new();
-    let layout = arrow::array::layout(arrow_array.data_type());
-    for (buffer, spec) in arrow_array.buffers().iter().zip(&layout.buffers) {
-        let len = buffer.len();
-        assert!(
-            target_buffer[*next_offset..].len() >= len,
-            "target buffer too small (total_len: {}, offset: {}, required_len: {len})",
-            target_buffer.len(),
-            *next_offset,
-        );
-        // add alignment padding
-        if let BufferSpec::FixedWidth { alignment, .. } = *spec {
-            *next_offset = (*next_offset).div_ceil(alignment) * alignment;
-        }
-
-        target_buffer[*next_offset..][..len].copy_from_slice(buffer.as_slice());
-        buffer_offsets.push(BufferOffset {
-            offset: *next_offset,
-            len,
-        });
-        *next_offset += len;
-    }
-
-    let mut child_data = Vec::new();
-    for child in arrow_array.child_data() {
-        let child_type_info = copy_array_into_sample_inner(target_buffer, next_offset, child);
-        child_data.push(child_type_info);
-    }
-
-    ArrowTypeInfo {
-        data_type: arrow_array.data_type().clone(),
-        len: arrow_array.len(),
-        null_count: arrow_array.null_count(),
-        validity: arrow_array.nulls().map(|b| b.validity().to_owned()),
-        offset: arrow_array.offset(),
-        buffer_offsets,
-        child_data,
-    }
-}
-
-/// Tries to convert the given raw Arrow buffer into an Arrow array.
-///
-/// The `type_info` is required for decoding the `raw_buffer` correctly.
-pub fn buffer_into_arrow_array(
-    raw_buffer: &arrow::buffer::Buffer,
-    type_info: &ArrowTypeInfo,
-) -> eyre::Result<arrow::array::ArrayData> {
-    if raw_buffer.is_empty() {
-        return Ok(arrow::array::ArrayData::new_empty(&type_info.data_type));
-    }
-
-    let mut buffers = Vec::new();
-    for BufferOffset { offset, len } in &type_info.buffer_offsets {
-        buffers.push(raw_buffer.slice_with_length(*offset, *len));
-    }
-
-    let mut child_data = Vec::new();
-    for child_type_info in &type_info.child_data {
-        child_data.push(buffer_into_arrow_array(raw_buffer, child_type_info)?)
-    }
-
-    arrow::array::ArrayData::try_new(
-        type_info.data_type.clone(),
-        type_info.len,
-        type_info
-            .validity
-            .clone()
-            .map(arrow::buffer::Buffer::from_vec),
-        type_info.offset,
-        buffers,
-        child_data,
-    )
-    .context("Error creating Arrow array")
-}

apis/rust/node/src/node/control_channel.rs

@@ -1,116 +0,0 @@
-use std::sync::Arc;
-
-use crate::daemon_connection::DaemonChannel;
-use dora_core::{
-    config::{DataId, NodeId},
-    uhlc::HLC,
-};
-use dora_message::{
-    DataflowId,
-    daemon_to_node::{DaemonCommunication, DaemonReply},
-    metadata::Metadata,
-    node_to_daemon::{DaemonRequest, DataMessage, Timestamped},
-};
-use eyre::{Context, bail, eyre};
-
-pub(crate) struct ControlChannel {
-    channel: DaemonChannel,
-    clock: Arc<HLC>,
-}
-
-impl ControlChannel {
-    #[tracing::instrument(level = "trace", skip(clock))]
-    pub(crate) fn init(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        daemon_communication: &DaemonCommunication,
-        clock: Arc<HLC>,
-    ) -> eyre::Result<Self> {
-        let channel = match daemon_communication {
-            DaemonCommunication::Shmem {
-                daemon_control_region_id,
-                ..
-            } => unsafe { DaemonChannel::new_shmem(daemon_control_region_id) }
-                .wrap_err("failed to create shmem control channel")?,
-            DaemonCommunication::Tcp { socket_addr } => DaemonChannel::new_tcp(*socket_addr)
-                .wrap_err("failed to connect control channel")?,
-            #[cfg(unix)]
-            DaemonCommunication::UnixDomain { socket_file } => {
-                DaemonChannel::new_unix_socket(socket_file)
-                    .wrap_err("failed to connect control channel")?
-            }
-        };
-
-        Self::init_on_channel(dataflow_id, node_id, channel, clock)
-    }
-
-    #[tracing::instrument(skip(channel, clock), level = "trace")]
-    pub fn init_on_channel(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        mut channel: DaemonChannel,
-        clock: Arc<HLC>,
-    ) -> eyre::Result<Self> {
-        channel.register(dataflow_id, node_id.clone(), clock.new_timestamp())?;
-
-        Ok(Self { channel, clock })
-    }
-
-    pub fn report_outputs_done(&mut self) -> eyre::Result<()> {
-        let reply = self
-            .channel
-            .request(&Timestamped {
-                inner: DaemonRequest::OutputsDone,
-                timestamp: self.clock.new_timestamp(),
-            })
-            .wrap_err("failed to report outputs done to dora-daemon")?;
-        match reply {
-            DaemonReply::Result(result) => result
-                .map_err(|e| eyre!(e))
-                .wrap_err("failed to report outputs done event to dora-daemon")?,
-            other => bail!("unexpected outputs done reply: {other:?}"),
-        }
-        Ok(())
-    }
-
-    pub fn report_closed_outputs(&mut self, outputs: Vec<DataId>) -> eyre::Result<()> {
-        let reply = self
-            .channel
-            .request(&Timestamped {
-                inner: DaemonRequest::CloseOutputs(outputs),
-                timestamp: self.clock.new_timestamp(),
-            })
-            .wrap_err("failed to report closed outputs to dora-daemon")?;
-        match reply {
-            DaemonReply::Result(result) => result
-                .map_err(|e| eyre!(e))
-                .wrap_err("failed to receive closed outputs reply from dora-daemon")?,
-            other => bail!("unexpected closed outputs reply: {other:?}"),
-        }
-        Ok(())
-    }
-
-    pub fn send_message(
-        &mut self,
-        output_id: DataId,
-        metadata: Metadata,
-        data: Option<DataMessage>,
-    ) -> eyre::Result<()> {
-        let request = DaemonRequest::SendMessage {
-            output_id,
-            metadata,
-            data,
-        };
-        let reply = self
-            .channel
-            .request(&Timestamped {
-                inner: request,
-                timestamp: self.clock.new_timestamp(),
-            })
-            .wrap_err("failed to send SendMessage request to dora-daemon")?;
-        match reply {
-            DaemonReply::Empty => Ok(()),
-            other => bail!("unexpected SendMessage reply: {other:?}"),
-        }
-    }
-}

apis/rust/node/src/node/drop_stream.rs

@@ -1,182 +0,0 @@
-use std::{sync::Arc, time::Duration};
-
-use crate::daemon_connection::DaemonChannel;
-use dora_core::{config::NodeId, uhlc};
-use dora_message::{
-    DataflowId,
-    daemon_to_node::{DaemonCommunication, DaemonReply, NodeDropEvent},
-    node_to_daemon::{DaemonRequest, DropToken, Timestamped},
-};
-use eyre::{Context, eyre};
-use flume::RecvTimeoutError;
-
-pub struct DropStream {
-    receiver: flume::Receiver<DropToken>,
-    _thread_handle: DropStreamThreadHandle,
-}
-
-impl DropStream {
-    #[tracing::instrument(level = "trace", skip(hlc))]
-    pub(crate) fn init(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        daemon_communication: &DaemonCommunication,
-        hlc: Arc<uhlc::HLC>,
-    ) -> eyre::Result<Self> {
-        let channel = match daemon_communication {
-            DaemonCommunication::Shmem {
-                daemon_drop_region_id,
-                ..
-            } => {
-                unsafe { DaemonChannel::new_shmem(daemon_drop_region_id) }.wrap_err_with(|| {
-                    format!("failed to create shmem drop stream for node `{node_id}`")
-                })?
-            }
-            DaemonCommunication::Tcp { socket_addr } => DaemonChannel::new_tcp(*socket_addr)
-                .wrap_err_with(|| format!("failed to connect drop stream for node `{node_id}`"))?,
-            #[cfg(unix)]
-            DaemonCommunication::UnixDomain { socket_file } => {
-                DaemonChannel::new_unix_socket(socket_file).wrap_err_with(|| {
-                    format!("failed to connect drop stream for node `{node_id}`")
-                })?
-            }
-        };
-
-        Self::init_on_channel(dataflow_id, node_id, channel, hlc)
-    }
-
-    pub fn init_on_channel(
-        dataflow_id: DataflowId,
-        node_id: &NodeId,
-        mut channel: DaemonChannel,
-        clock: Arc<uhlc::HLC>,
-    ) -> eyre::Result<Self> {
-        channel.register(dataflow_id, node_id.clone(), clock.new_timestamp())?;
-
-        let reply = channel
-            .request(&Timestamped {
-                inner: DaemonRequest::SubscribeDrop,
-                timestamp: clock.new_timestamp(),
-            })
-            .map_err(|e| eyre!(e))
-            .wrap_err("failed to create subscription with dora-daemon")?;
-
-        match reply {
-            DaemonReply::Result(Ok(())) => {}
-            DaemonReply::Result(Err(err)) => {
-                eyre::bail!("drop subscribe failed: {err}")
-            }
-            other => eyre::bail!("unexpected drop subscribe reply: {other:?}"),
-        }
-
-        let (tx, rx) = flume::bounded(0);
-        let node_id_cloned = node_id.clone();
-
-        let handle = std::thread::spawn(|| drop_stream_loop(node_id_cloned, tx, channel, clock));
-
-        Ok(Self {
-            receiver: rx,
-            _thread_handle: DropStreamThreadHandle::new(node_id.clone(), handle),
-        })
-    }
-}
-
-impl std::ops::Deref for DropStream {
-    type Target = flume::Receiver<DropToken>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.receiver
-    }
-}
-
-#[tracing::instrument(skip(tx, channel, clock))]
-fn drop_stream_loop(
-    node_id: NodeId,
-    tx: flume::Sender<DropToken>,
-    mut channel: DaemonChannel,
-    clock: Arc<uhlc::HLC>,
-) {
-    'outer: loop {
-        let daemon_request = Timestamped {
-            inner: DaemonRequest::NextFinishedDropTokens,
-            timestamp: clock.new_timestamp(),
-        };
-        let events = match channel.request(&daemon_request) {
-            Ok(DaemonReply::NextDropEvents(events)) => {
-                if events.is_empty() {
-                    tracing::trace!("drop stream closed for node `{node_id}`");
-                    break;
-                } else {
-                    events
-                }
-            }
-            Ok(other) => {
-                let err = eyre!("unexpected drop reply: {other:?}");
-                tracing::warn!("{err:?}");
-                continue;
-            }
-            Err(err) => {
-                let err = eyre!(err).wrap_err("failed to receive incoming drop event");
-                tracing::warn!("{err:?}");
-                continue;
-            }
-        };
-        for Timestamped { inner, timestamp } in events {
-            if let Err(err) = clock.update_with_timestamp(&timestamp) {
-                tracing::warn!("failed to update HLC: {err}");
-            }
-            match inner {
-                NodeDropEvent::OutputDropped { drop_token } => {
-                    if tx.send(drop_token).is_err() {
-                        tracing::warn!(
-                            "drop channel was closed already, could not forward \
-                            drop token`{drop_token:?}`"
-                        );
-                        break 'outer;
-                    }
-                }
-            }
-        }
-    }
-}
-
-struct DropStreamThreadHandle {
-    node_id: NodeId,
-    handle: flume::Receiver<std::thread::Result<()>>,
-}
-
-impl DropStreamThreadHandle {
-    fn new(node_id: NodeId, join_handle: std::thread::JoinHandle<()>) -> Self {
-        let (tx, rx) = flume::bounded(1);
-        std::thread::spawn(move || {
-            let _ = tx.send(join_handle.join());
-        });
-        Self {
-            node_id,
-            handle: rx,
-        }
-    }
-}
-
-impl Drop for DropStreamThreadHandle {
-    #[tracing::instrument(skip(self), fields(node_id = %self.node_id))]
-    fn drop(&mut self) {
-        if self.handle.is_empty() {
-            tracing::trace!("waiting for drop stream thread");
-        }
-        match self.handle.recv_timeout(Duration::from_secs(2)) {
-            Ok(Ok(())) => {
-                tracing::trace!("drop stream thread done");
-            }
-            Ok(Err(_)) => {
-                tracing::error!("drop stream thread panicked");
-            }
-            Err(RecvTimeoutError::Timeout) => {
-                tracing::warn!("timeout while waiting for drop stream thread");
-            }
-            Err(RecvTimeoutError::Disconnected) => {
-                tracing::warn!("drop stream thread result channel closed unexpectedly");
-            }
-        }
-    }
-}

apis/rust/node/src/node/mod.rs

@@ -1,686 +0,0 @@
-use crate::{EventStream, daemon_connection::DaemonChannel};
-
-use self::{
-    arrow_utils::{copy_array_into_sample, required_data_size},
-    control_channel::ControlChannel,
-    drop_stream::DropStream,
-};
-use aligned_vec::{AVec, ConstAlign};
-use arrow::array::Array;
-use dora_core::{
-    config::{DataId, NodeId, NodeRunConfig},
-    descriptor::Descriptor,
-    metadata::ArrowTypeInfoExt,
-    topics::{DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT, LOCALHOST},
-    uhlc,
-};
-
-use dora_message::{
-    DataflowId,
-    daemon_to_node::{DaemonReply, NodeConfig},
-    metadata::{ArrowTypeInfo, Metadata, MetadataParameters},
-    node_to_daemon::{DaemonRequest, DataMessage, DropToken, Timestamped},
-};
-use eyre::{WrapErr, bail};
-use shared_memory_extended::{Shmem, ShmemConf};
-use std::{
-    collections::{BTreeSet, HashMap, VecDeque},
-    ops::{Deref, DerefMut},
-    sync::Arc,
-    time::Duration,
-};
-use tracing::{info, warn};
-
-#[cfg(feature = "metrics")]
-use dora_metrics::run_metrics_monitor;
-#[cfg(feature = "tracing")]
-use dora_tracing::TracingBuilder;
-
-use tokio::runtime::{Handle, Runtime};
-
-pub mod arrow_utils;
-mod control_channel;
-mod drop_stream;
-
-/// The data size threshold at which we start using shared memory.
-///
-/// Shared memory works by sharing memory pages. This means that the smallest
-/// memory region that can be shared is one memory page, which is typically
-/// 4KiB.
-///
-/// Using shared memory for messages smaller than the page size still requires
-/// sharing a full page, so we have some memory overhead. We also have some
-/// performance overhead because we need to issue multiple syscalls. For small
-/// messages it is faster to send them over a traditional TCP stream (or similar).
-///
-/// This hardcoded threshold value specifies which messages are sent through
-/// shared memory. Messages that are smaller than this threshold are sent through
-/// TCP.
-pub const ZERO_COPY_THRESHOLD: usize = 4096;
-
-#[allow(dead_code)]
-enum TokioRuntime {
-    Runtime(Runtime),
-    Handle(Handle),
-}
-
-/// Allows sending outputs and retrieving node information.
-///
-/// The main purpose of this struct is to send outputs via Dora. There are also functions available
-/// for retrieving the node configuration.
-pub struct DoraNode {
-    id: NodeId,
-    dataflow_id: DataflowId,
-    node_config: NodeRunConfig,
-    control_channel: ControlChannel,
-    clock: Arc<uhlc::HLC>,
-
-    sent_out_shared_memory: HashMap<DropToken, ShmemHandle>,
-    drop_stream: DropStream,
-    cache: VecDeque<ShmemHandle>,
-
-    dataflow_descriptor: serde_yaml::Result<Descriptor>,
-    warned_unknown_output: BTreeSet<DataId>,
-    _rt: TokioRuntime,
-}
-
-impl DoraNode {
-    /// Initiate a node from environment variables set by the Dora daemon.
-    ///
-    /// This is the recommended initialization function for Dora nodes, which are spawned by
-    /// Dora daemon instances.
-    ///
-    ///
-    /// ```no_run
-    /// use dora_node_api::DoraNode;
-    ///
-    /// let (mut node, mut events) = DoraNode::init_from_env().expect("Could not init node.");
-    /// ```
-    ///
-    pub fn init_from_env() -> eyre::Result<(Self, EventStream)> {
-        let node_config: NodeConfig = {
-            let raw = std::env::var("DORA_NODE_CONFIG").wrap_err(
-                "env variable DORA_NODE_CONFIG must be set. Are you sure your using `dora start`?",
-            )?;
-            serde_yaml::from_str(&raw).context("failed to deserialize node config")?
-        };
-        #[cfg(feature = "tracing")]
-        {
-            TracingBuilder::new(node_config.node_id.as_ref())
-                .build()
-                .wrap_err("failed to set up tracing subscriber")?;
-        }
-
-        Self::init(node_config)
-    }
-
-    /// Initiate a node from a dataflow id and a node id.
-    ///
-    /// This initialization function should be used for [_dynamic nodes_](index.html#dynamic-nodes).
-    ///
-    /// ```no_run
-    /// use dora_node_api::DoraNode;
-    /// use dora_node_api::dora_core::config::NodeId;
-    ///
-    /// let (mut node, mut events) = DoraNode::init_from_node_id(NodeId::from("plot".to_string())).expect("Could not init node plot");
-    /// ```
-    ///
-    pub fn init_from_node_id(node_id: NodeId) -> eyre::Result<(Self, EventStream)> {
-        // Make sure that the node is initialized outside of dora start.
-        let daemon_address = (LOCALHOST, DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT).into();
-
-        let mut channel =
-            DaemonChannel::new_tcp(daemon_address).context("Could not connect to the daemon")?;
-        let clock = Arc::new(uhlc::HLC::default());
-
-        let reply = channel
-            .request(&Timestamped {
-                inner: DaemonRequest::NodeConfig { node_id },
-                timestamp: clock.new_timestamp(),
-            })
-            .wrap_err("failed to request node config from daemon")?;
-        match reply {
-            DaemonReply::NodeConfig {
-                result: Ok(node_config),
-            } => Self::init(node_config),
-            DaemonReply::NodeConfig { result: Err(error) } => {
-                bail!("failed to get node config from daemon: {error}")
-            }
-            _ => bail!("unexpected reply from daemon"),
-        }
-    }
-
-    /// Dynamic initialization function for nodes that are sometimes used as dynamic nodes.
-    ///
-    /// This function first tries initializing the traditional way through
-    /// [`init_from_env`][Self::init_from_env]. If this fails, it falls back to
-    /// [`init_from_node_id`][Self::init_from_node_id].
-    pub fn init_flexible(node_id: NodeId) -> eyre::Result<(Self, EventStream)> {
-        if std::env::var("DORA_NODE_CONFIG").is_ok() {
-            info!(
-                "Skipping {node_id} specified within the node initialization in favor of `DORA_NODE_CONFIG` specified by `dora start`"
-            );
-            Self::init_from_env()
-        } else {
-            Self::init_from_node_id(node_id)
-        }
-    }
-
-    /// Internal initialization routine that should not be used outside of Dora.
-    #[doc(hidden)]
-    #[tracing::instrument]
-    pub fn init(node_config: NodeConfig) -> eyre::Result<(Self, EventStream)> {
-        let NodeConfig {
-            dataflow_id,
-            node_id,
-            run_config,
-            daemon_communication,
-            dataflow_descriptor,
-            dynamic: _,
-        } = node_config;
-        let clock = Arc::new(uhlc::HLC::default());
-        let input_config = run_config.inputs.clone();
-
-        let rt = match Handle::try_current() {
-            Ok(handle) => TokioRuntime::Handle(handle),
-            Err(_) => TokioRuntime::Runtime(
-                tokio::runtime::Builder::new_multi_thread()
-                    .worker_threads(2)
-                    .enable_all()
-                    .build()
-                    .context("tokio runtime failed")?,
-            ),
-        };
-
-        #[cfg(feature = "metrics")]
-        {
-            let id = format!("{dataflow_id}/{node_id}");
-            let monitor_task = async move {
-                if let Err(e) = run_metrics_monitor(id.clone())
-                    .await
-                    .wrap_err("metrics monitor exited unexpectedly")
-                {
-                    warn!("metrics monitor failed: {:#?}", e);
-                }
-            };
-            match &rt {
-                TokioRuntime::Runtime(rt) => rt.spawn(monitor_task),
-                TokioRuntime::Handle(handle) => handle.spawn(monitor_task),
-            };
-        }
-
-        let event_stream = EventStream::init(
-            dataflow_id,
-            &node_id,
-            &daemon_communication,
-            input_config,
-            clock.clone(),
-        )
-        .wrap_err("failed to init event stream")?;
-        let drop_stream =
-            DropStream::init(dataflow_id, &node_id, &daemon_communication, clock.clone())
-                .wrap_err("failed to init drop stream")?;
-        let control_channel =
-            ControlChannel::init(dataflow_id, &node_id, &daemon_communication, clock.clone())
-                .wrap_err("failed to init control channel")?;
-
-        let node = Self {
-            id: node_id,
-            dataflow_id,
-            node_config: run_config.clone(),
-            control_channel,
-            clock,
-            sent_out_shared_memory: HashMap::new(),
-            drop_stream,
-            cache: VecDeque::new(),
-            dataflow_descriptor: serde_yaml::from_value(dataflow_descriptor),
-            warned_unknown_output: BTreeSet::new(),
-            _rt: rt,
-        };
-        Ok((node, event_stream))
-    }
-
-    fn validate_output(&mut self, output_id: &DataId) -> bool {
-        if !self.node_config.outputs.contains(output_id) {
-            if !self.warned_unknown_output.contains(output_id) {
-                warn!("Ignoring output `{output_id}` not in node's output list.");
-                self.warned_unknown_output.insert(output_id.clone());
-            }
-            false
-        } else {
-            true
-        }
-    }
-
-    /// Send raw data from the node to the other nodes.
-    ///
-    /// We take a closure as an input to enable zero copy on send.
-    ///
-    /// ```no_run
-    /// use dora_node_api::{DoraNode, MetadataParameters};
-    /// use dora_core::config::DataId;
-    ///
-    /// let (mut node, mut events) = DoraNode::init_from_env().expect("Could not init node.");
-    ///
-    /// let output = DataId::from("output_id".to_owned());
-    ///
-    /// let data: &[u8] = &[0, 1, 2, 3];
-    /// let parameters = MetadataParameters::default();
-    ///
-    /// node.send_output_raw(
-    ///    output,
-    ///    parameters,
-    ///    data.len(),
-    ///    |out| {
-    ///         out.copy_from_slice(data);
-    ///     }).expect("Could not send output");
-    /// ```
-    ///
-    /// Ignores the output if the given `output_id` is not specified as node output in the dataflow
-    /// configuration file.
-    pub fn send_output_raw<F>(
-        &mut self,
-        output_id: DataId,
-        parameters: MetadataParameters,
-        data_len: usize,
-        data: F,
-    ) -> eyre::Result<()>
-    where
-        F: FnOnce(&mut [u8]),
-    {
-        if !self.validate_output(&output_id) {
-            return Ok(());
-        };
-        let mut sample = self.allocate_data_sample(data_len)?;
-        data(&mut sample);
-
-        let type_info = ArrowTypeInfo::byte_array(data_len);
-
-        self.send_output_sample(output_id, type_info, parameters, Some(sample))
-    }
-
-    /// Sends the give Arrow array as an output message.
-    ///
-    /// Uses shared memory for efficient data transfer if suitable.
-    ///
-    /// This method might copy the message once to move it to shared memory.
-    ///    
-    /// Ignores the output if the given `output_id` is not specified as node output in the dataflow
-    /// configuration file.
-    pub fn send_output(
-        &mut self,
-        output_id: DataId,
-        parameters: MetadataParameters,
-        data: impl Array,
-    ) -> eyre::Result<()> {
-        if !self.validate_output(&output_id) {
-            return Ok(());
-        };
-
-        let arrow_array = data.to_data();
-
-        let total_len = required_data_size(&arrow_array);
-
-        let mut sample = self.allocate_data_sample(total_len)?;
-        let type_info = copy_array_into_sample(&mut sample, &arrow_array);
-
-        self.send_output_sample(output_id, type_info, parameters, Some(sample))
-            .wrap_err("failed to send output")?;
-
-        Ok(())
-    }
-
-    /// Send the given raw byte data as output.
-    ///
-    /// Might copy the data once to move it into shared memory.
-    ///
-    /// Ignores the output if the given `output_id` is not specified as node output in the dataflow
-    /// configuration file.
-    pub fn send_output_bytes(
-        &mut self,
-        output_id: DataId,
-        parameters: MetadataParameters,
-        data_len: usize,
-        data: &[u8],
-    ) -> eyre::Result<()> {
-        if !self.validate_output(&output_id) {
-            return Ok(());
-        };
-        self.send_output_raw(output_id, parameters, data_len, |sample| {
-            sample.copy_from_slice(data)
-        })
-    }
-
-    /// Send the give raw byte data with the provided type information.
-    ///
-    /// It is recommended to use a function like [`send_output`][Self::send_output] instead.
-    ///
-    /// Ignores the output if the given `output_id` is not specified as node output in the dataflow
-    /// configuration file.
-    pub fn send_typed_output<F>(
-        &mut self,
-        output_id: DataId,
-        type_info: ArrowTypeInfo,
-        parameters: MetadataParameters,
-        data_len: usize,
-        data: F,
-    ) -> eyre::Result<()>
-    where
-        F: FnOnce(&mut [u8]),
-    {
-        if !self.validate_output(&output_id) {
-            return Ok(());
-        };
-
-        let mut sample = self.allocate_data_sample(data_len)?;
-        data(&mut sample);
-
-        self.send_output_sample(output_id, type_info, parameters, Some(sample))
-    }
-
-    /// Sends the given [`DataSample`] as output, combined with the given type information.
-    ///
-    /// It is recommended to use a function like [`send_output`][Self::send_output] instead.
-    ///
-    /// Ignores the output if the given `output_id` is not specified as node output in the dataflow
-    /// configuration file.
-    pub fn send_output_sample(
-        &mut self,
-        output_id: DataId,
-        type_info: ArrowTypeInfo,
-        parameters: MetadataParameters,
-        sample: Option<DataSample>,
-    ) -> eyre::Result<()> {
-        self.handle_finished_drop_tokens()?;
-
-        let metadata = Metadata::from_parameters(self.clock.new_timestamp(), type_info, parameters);
-
-        let (data, shmem) = match sample {
-            Some(sample) => sample.finalize(),
-            None => (None, None),
-        };
-
-        self.control_channel
-            .send_message(output_id.clone(), metadata, data)
-            .wrap_err_with(|| format!("failed to send output {output_id}"))?;
-
-        if let Some((shared_memory, drop_token)) = shmem {
-            self.sent_out_shared_memory
-                .insert(drop_token, shared_memory);
-        }
-
-        Ok(())
-    }
-
-    /// Report the given outputs IDs as closed.
-    ///
-    /// The node is not allowed to send more outputs with the closed IDs.
-    ///
-    /// Closing outputs early can be helpful to receivers.
-    pub fn close_outputs(&mut self, outputs_ids: Vec<DataId>) -> eyre::Result<()> {
-        for output_id in &outputs_ids {
-            if !self.node_config.outputs.remove(output_id) {
-                eyre::bail!("unknown output {output_id}");
-            }
-        }
-
-        self.control_channel
-            .report_closed_outputs(outputs_ids)
-            .wrap_err("failed to report closed outputs to daemon")?;
-
-        Ok(())
-    }
-
-    /// Returns the ID of the node as specified in the dataflow configuration file.
-    pub fn id(&self) -> &NodeId {
-        &self.id
-    }
-
-    /// Returns the unique identifier for the running dataflow instance.
-    ///
-    /// Dora assigns each dataflow instance a random identifier when started.
-    pub fn dataflow_id(&self) -> &DataflowId {
-        &self.dataflow_id
-    }
-
-    /// Returns the input and output configuration of this node.
-    pub fn node_config(&self) -> &NodeRunConfig {
-        &self.node_config
-    }
-
-    /// Allocates a [`DataSample`] of the specified size.
-    ///
-    /// The data sample will use shared memory when suitable to enable efficient data transfer
-    /// when sending an output message.
-    pub fn allocate_data_sample(&mut self, data_len: usize) -> eyre::Result<DataSample> {
-        let data = if data_len >= ZERO_COPY_THRESHOLD {
-            // create shared memory region
-            let shared_memory = self.allocate_shared_memory(data_len)?;
-
-            DataSample {
-                inner: DataSampleInner::Shmem(shared_memory),
-                len: data_len,
-            }
-        } else {
-            let avec: AVec<u8, ConstAlign<128>> = AVec::__from_elem(128, 0, data_len);
-
-            avec.into()
-        };
-
-        Ok(data)
-    }
-
-    fn allocate_shared_memory(&mut self, data_len: usize) -> eyre::Result<ShmemHandle> {
-        let cache_index = self
-            .cache
-            .iter()
-            .enumerate()
-            .rev()
-            .filter(|(_, s)| s.len() >= data_len)
-            .min_by_key(|(_, s)| s.len())
-            .map(|(i, _)| i);
-        let memory = match cache_index {
-            Some(i) => {
-                // we know that this index exists, so we can safely unwrap here
-                self.cache.remove(i).unwrap()
-            }
-            None => ShmemHandle(Box::new(
-                ShmemConf::new()
-                    .size(data_len)
-                    .writable(true)
-                    .create()
-                    .wrap_err("failed to allocate shared memory")?,
-            )),
-        };
-        assert!(memory.len() >= data_len);
-
-        Ok(memory)
-    }
-
-    fn handle_finished_drop_tokens(&mut self) -> eyre::Result<()> {
-        loop {
-            match self.drop_stream.try_recv() {
-                Ok(token) => match self.sent_out_shared_memory.remove(&token) {
-                    Some(region) => self.add_to_cache(region),
-                    None => tracing::warn!("received unknown finished drop token `{token:?}`"),
-                },
-                Err(flume::TryRecvError::Empty) => break,
-                Err(flume::TryRecvError::Disconnected) => {
-                    bail!("event stream was closed before sending all expected drop tokens")
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn add_to_cache(&mut self, memory: ShmemHandle) {
-        const MAX_CACHE_SIZE: usize = 20;
-
-        self.cache.push_back(memory);
-        while self.cache.len() > MAX_CACHE_SIZE {
-            self.cache.pop_front();
-        }
-    }
-
-    /// Returns the full dataflow descriptor that this node is part of.
-    ///
-    /// This method returns the parsed dataflow YAML file.
-    pub fn dataflow_descriptor(&self) -> eyre::Result<&Descriptor> {
-        match &self.dataflow_descriptor {
-            Ok(d) => Ok(d),
-            Err(err) => eyre::bail!(
-                "failed to parse dataflow descriptor: {err}\n\n
-                This might be caused by mismatched version numbers of dora \
-                daemon and the dora node API"
-            ),
-        }
-    }
-}
-
-impl Drop for DoraNode {
-    #[tracing::instrument(skip(self), fields(self.id = %self.id), level = "trace")]
-    fn drop(&mut self) {
-        // close all outputs first to notify subscribers as early as possible
-        if let Err(err) = self
-            .control_channel
-            .report_closed_outputs(
-                std::mem::take(&mut self.node_config.outputs)
-                    .into_iter()
-                    .collect(),
-            )
-            .context("failed to close outputs on drop")
-        {
-            tracing::warn!("{err:?}")
-        }
-
-        while !self.sent_out_shared_memory.is_empty() {
-            if self.drop_stream.is_empty() {
-                tracing::trace!(
-                    "waiting for {} remaining drop tokens",
-                    self.sent_out_shared_memory.len()
-                );
-            }
-
-            match self.drop_stream.recv_timeout(Duration::from_secs(2)) {
-                Ok(token) => {
-                    self.sent_out_shared_memory.remove(&token);
-                }
-                Err(flume::RecvTimeoutError::Disconnected) => {
-                    tracing::warn!(
-                        "finished_drop_tokens channel closed while still waiting for drop tokens; \
-                        closing {} shared memory regions that might not yet been mapped.",
-                        self.sent_out_shared_memory.len()
-                    );
-                    break;
-                }
-                Err(flume::RecvTimeoutError::Timeout) => {
-                    tracing::warn!(
-                        "timeout while waiting for drop tokens; \
-                        closing {} shared memory regions that might not yet been mapped.",
-                        self.sent_out_shared_memory.len()
-                    );
-                    break;
-                }
-            }
-        }
-
-        if let Err(err) = self.control_channel.report_outputs_done() {
-            tracing::warn!("{err:?}")
-        }
-    }
-}
-
-/// A data region suitable for sending as an output message.
-///
-/// The region is stored in shared memory when suitable to enable efficient data transfer.
-///
-/// `DataSample` implements the [`Deref`] and [`DerefMut`] traits to read and write the mapped data.
-pub struct DataSample {
-    inner: DataSampleInner,
-    len: usize,
-}
-
-impl DataSample {
-    fn finalize(self) -> (Option<DataMessage>, Option<(ShmemHandle, DropToken)>) {
-        match self.inner {
-            DataSampleInner::Shmem(shared_memory) => {
-                let drop_token = DropToken::generate();
-                let data = DataMessage::SharedMemory {
-                    shared_memory_id: shared_memory.get_os_id().to_owned(),
-                    len: self.len,
-                    drop_token,
-                };
-                (Some(data), Some((shared_memory, drop_token)))
-            }
-            DataSampleInner::Vec(buffer) => (Some(DataMessage::Vec(buffer)), None),
-        }
-    }
-}
-
-impl Deref for DataSample {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        let slice = match &self.inner {
-            DataSampleInner::Shmem(handle) => unsafe { handle.as_slice() },
-            DataSampleInner::Vec(data) => data,
-        };
-        &slice[..self.len]
-    }
-}
-
-impl DerefMut for DataSample {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        let slice = match &mut self.inner {
-            DataSampleInner::Shmem(handle) => unsafe { handle.as_slice_mut() },
-            DataSampleInner::Vec(data) => data,
-        };
-        &mut slice[..self.len]
-    }
-}
-
-impl From<AVec<u8, ConstAlign<128>>> for DataSample {
-    fn from(value: AVec<u8, ConstAlign<128>>) -> Self {
-        Self {
-            len: value.len(),
-            inner: DataSampleInner::Vec(value),
-        }
-    }
-}
-
-impl std::fmt::Debug for DataSample {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let kind = match &self.inner {
-            DataSampleInner::Shmem(_) => "SharedMemory",
-            DataSampleInner::Vec(_) => "Vec",
-        };
-        f.debug_struct("DataSample")
-            .field("len", &self.len)
-            .field("kind", &kind)
-            .finish_non_exhaustive()
-    }
-}
-
-enum DataSampleInner {
-    Shmem(ShmemHandle),
-    Vec(AVec<u8, ConstAlign<128>>),
-}
-
-struct ShmemHandle(Box<Shmem>);
-
-impl Deref for ShmemHandle {
-    type Target = Shmem;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for ShmemHandle {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-unsafe impl Send for ShmemHandle {}
-unsafe impl Sync for ShmemHandle {}

apis/rust/operator/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "dora-operator-api"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-dora-operator-api-macros = { workspace = true }
-dora-operator-api-types = { workspace = true }
-dora-arrow-convert = { workspace = true }

apis/rust/operator/macros/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "dora-operator-api-macros"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-description = "Rust API Macros for Dora Operator"
-documentation.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-proc-macro = true
-
-[dependencies]
-syn = { version = "1.0.81", features = ["full"] }
-quote = "1.0.10"
-proc-macro2 = "1.0.32"

apis/rust/operator/macros/src/lib.rs

@@ -1,74 +0,0 @@
-use proc_macro::TokenStream;
-use proc_macro2::TokenStream as TokenStream2;
-use quote::quote;
-
-extern crate proc_macro;
-
-#[proc_macro]
-pub fn register_operator(item: TokenStream) -> TokenStream {
-    // convert from `TokenStream` to `TokenStream2`, which is used by the
-    // `syn` crate
-    let item = TokenStream2::from(item);
-    // generate the dora wrapper functions
-    let generated = register_operator_impl(&item).unwrap_or_else(|err| err.to_compile_error());
-    // output the generated functions
-    let tokens = quote! {
-        #generated
-    };
-    // convert the type back from `TokenStream2` to `TokenStream`
-    tokens.into()
-}
-
-/// Generates the wrapper functions for the annotated function.
-fn register_operator_impl(item: &TokenStream2) -> syn::Result<TokenStream2> {
-    // parse the type given to the `register_operator` macro
-    let operator_ty: syn::TypePath = syn::parse2(item.clone())
-        .map_err(|e| syn::Error::new(e.span(), "expected type as argument"))?;
-
-    let init = quote! {
-        #[unsafe(no_mangle)]
-        pub unsafe extern "C" fn dora_init_operator() -> dora_operator_api::types::DoraInitResult {
-            dora_operator_api::raw::dora_init_operator::<#operator_ty>()
-        }
-
-        const _DORA_INIT_OPERATOR: dora_operator_api::types::DoraInitOperator = dora_operator_api::types::DoraInitOperator {
-            init_operator: dora_init_operator,
-        };
-    };
-
-    let drop = quote! {
-        #[unsafe(no_mangle)]
-        pub unsafe extern "C" fn dora_drop_operator(operator_context: *mut std::ffi::c_void)
-            -> dora_operator_api::types::DoraResult
-        {
-            dora_operator_api::raw::dora_drop_operator::<#operator_ty>(operator_context)
-        }
-
-        const _DORA_DROP_OPERATOR: dora_operator_api::types::DoraDropOperator = dora_operator_api::types::DoraDropOperator {
-            drop_operator: dora_drop_operator,
-        };
-    };
-
-    let on_event = quote! {
-        #[unsafe(no_mangle)]
-        pub unsafe extern "C" fn dora_on_event(
-            event: &mut dora_operator_api::types::RawEvent,
-            send_output: &dora_operator_api::types::SendOutput,
-            operator_context: *mut std::ffi::c_void,
-        ) -> dora_operator_api::types::OnEventResult {
-            dora_operator_api::raw::dora_on_event::<#operator_ty>(
-                event, send_output, operator_context
-            )
-        }
-
-        const _DORA_ON_EVENT: dora_operator_api::types::DoraOnEvent = dora_operator_api::types::DoraOnEvent {
-            on_event: dora_operator_api::types::OnEventFn(dora_on_event),
-        };
-    };
-
-    Ok(quote! {
-        #init
-        #drop
-        #on_event
-    })
-}

apis/rust/operator/src/lib.rs

@@ -1,69 +0,0 @@
-//! The operator API is a framework to implement dora operators.
-//! The implemented operator will be managed by `dora`.
-//!
-//! This framework enable us to make optimisation and provide advanced features.
-//! It is the recommended way of using `dora`.
-//!
-//! An operator requires to be registered and implement the `DoraOperator` trait.
-//! It is composed of an `on_event` method that defines the behaviour
-//! of the operator when there is an event such as receiving an input for example.
-//!
-//! Try it out with:
-//!
-//! ```bash
-//! dora new op --kind operator
-//! ```
-//!
-
-#![warn(unsafe_op_in_unsafe_fn)]
-#![allow(clippy::missing_safety_doc)]
-
-pub use dora_arrow_convert::*;
-pub use dora_operator_api_macros::register_operator;
-pub use dora_operator_api_types as types;
-pub use types::DoraStatus;
-use types::{
-    Metadata, Output, SendOutput,
-    arrow::{self, array::Array},
-};
-
-pub mod raw;
-
-#[derive(Debug)]
-#[non_exhaustive]
-pub enum Event<'a> {
-    Input { id: &'a str, data: ArrowData },
-    InputParseError { id: &'a str, error: String },
-    InputClosed { id: &'a str },
-    Stop,
-}
-
-pub trait DoraOperator: Default {
-    #[allow(clippy::result_unit_err)] // we use a () error type only for testing
-    fn on_event(
-        &mut self,
-        event: &Event,
-        output_sender: &mut DoraOutputSender,
-    ) -> Result<DoraStatus, String>;
-}
-
-pub struct DoraOutputSender<'a>(&'a SendOutput);
-
-impl DoraOutputSender<'_> {
-    ///  Send an output from the operator:
-    ///  - `id` is the `output_id` as defined in your dataflow.
-    ///  - `data` is the data that should be sent
-    pub fn send(&mut self, id: String, data: impl Array) -> Result<(), String> {
-        let (data_array, schema) =
-            arrow::ffi::to_ffi(&data.into_data()).map_err(|err| err.to_string())?;
-        let result = self.0.send_output.call(Output {
-            id: id.into(),
-            data_array,
-            schema,
-            metadata: Metadata {
-                open_telemetry_context: String::new().into(), // TODO
-            },
-        });
-        result.into_result()
-    }
-}

apis/rust/operator/src/raw.rs

@@ -1,80 +0,0 @@
-use crate::{DoraOperator, DoraOutputSender, DoraStatus, Event};
-use dora_operator_api_types::{
-    DoraInitResult, DoraResult, OnEventResult, RawEvent, SendOutput, arrow,
-};
-use std::ffi::c_void;
-
-pub type OutputFnRaw = unsafe extern "C" fn(
-    id_start: *const u8,
-    id_len: usize,
-    data_start: *const u8,
-    data_len: usize,
-    output_context: *const c_void,
-) -> isize;
-
-pub unsafe fn dora_init_operator<O: DoraOperator>() -> DoraInitResult {
-    let operator: O = Default::default();
-    let ptr: *mut O = Box::leak(Box::new(operator));
-    let operator_context: *mut c_void = ptr.cast();
-    DoraInitResult {
-        result: DoraResult { error: None },
-        operator_context,
-    }
-}
-
-pub unsafe fn dora_drop_operator<O>(operator_context: *mut c_void) -> DoraResult {
-    let raw: *mut O = operator_context.cast();
-    drop(unsafe { Box::from_raw(raw) });
-    DoraResult { error: None }
-}
-
-pub unsafe fn dora_on_event<O: DoraOperator>(
-    event: &mut RawEvent,
-    send_output: &SendOutput,
-    operator_context: *mut std::ffi::c_void,
-) -> OnEventResult {
-    let mut output_sender = DoraOutputSender(send_output);
-
-    let operator: &mut O = unsafe { &mut *operator_context.cast() };
-
-    let event_variant = if let Some(input) = &mut event.input {
-        let Some(data_array) = input.data_array.take() else {
-            return OnEventResult {
-                result: DoraResult::from_error("data already taken".to_string()),
-                status: DoraStatus::Continue,
-            };
-        };
-        let data = unsafe { arrow::ffi::from_ffi(data_array, &input.schema) };
-
-        match data {
-            Ok(data) => Event::Input {
-                id: &input.id,
-                data: arrow::array::make_array(data).into(),
-            },
-            Err(err) => Event::InputParseError {
-                id: &input.id,
-                error: format!("{err}"),
-            },
-        }
-    } else if let Some(input_id) = &event.input_closed {
-        Event::InputClosed { id: input_id }
-    } else if event.stop {
-        Event::Stop
-    } else {
-        // ignore unknown events
-        return OnEventResult {
-            result: DoraResult { error: None },
-            status: DoraStatus::Continue,
-        };
-    };
-    match operator.on_event(&event_variant, &mut output_sender) {
-        Ok(status) => OnEventResult {
-            result: DoraResult { error: None },
-            status,
-        },
-        Err(error) => OnEventResult {
-            result: DoraResult::from_error(error),
-            status: DoraStatus::Stop,
-        },
-    }
-}

apis/rust/operator/types/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "dora-operator-api-types"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-documentation.workspace = true
-description.workspace = true
-license.workspace = true
-repository.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-arrow = { workspace = true, features = ["ffi"] }
-dora-arrow-convert = { workspace = true }
-
-[dependencies.safer-ffi]
-version = "0.1.4"
-features = ["headers", "inventory-0-3-1"]

apis/rust/operator/types/src/lib.rs

@@ -1,212 +0,0 @@
-#![deny(elided_lifetimes_in_paths)] // required for safer-ffi
-#![allow(improper_ctypes_definitions)]
-#![allow(clippy::missing_safety_doc)]
-
-pub use arrow;
-use dora_arrow_convert::{ArrowData, IntoArrow};
-pub use safer_ffi;
-
-use arrow::{
-    array::Array,
-    ffi::{FFI_ArrowArray, FFI_ArrowSchema},
-};
-use core::slice;
-use safer_ffi::{
-    char_p::{self, char_p_boxed},
-    closure::ArcDynFn1,
-    derive_ReprC, ffi_export,
-};
-use std::{ops::Deref, path::Path};
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-pub struct DoraInitOperator {
-    pub init_operator: unsafe extern "C" fn() -> DoraInitResult,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-#[derive(Debug)]
-pub struct DoraInitResult {
-    pub result: DoraResult,
-    pub operator_context: *mut std::ffi::c_void,
-}
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-pub struct DoraDropOperator {
-    pub drop_operator: unsafe extern "C" fn(operator_context: *mut std::ffi::c_void) -> DoraResult,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-#[derive(Debug)]
-pub struct DoraResult {
-    pub error: Option<safer_ffi::boxed::Box<safer_ffi::String>>,
-}
-
-impl DoraResult {
-    pub const SUCCESS: Self = Self { error: None };
-
-    pub fn from_error(error: String) -> Self {
-        Self {
-            error: Some(Box::new(safer_ffi::String::from(error)).into()),
-        }
-    }
-
-    pub fn error(&self) -> Option<&str> {
-        self.error.as_deref().map(|s| s.deref())
-    }
-
-    pub fn into_result(self) -> Result<(), String> {
-        match self.error {
-            None => Ok(()),
-            Some(error) => {
-                let converted = safer_ffi::boxed::Box_::into(error);
-                Err((*converted).into())
-            }
-        }
-    }
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-pub struct DoraOnEvent {
-    pub on_event: OnEventFn,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(transparent)]
-pub struct OnEventFn(
-    pub  unsafe extern "C" fn(
-        event: &mut RawEvent,
-        send_output: &SendOutput,
-        operator_context: *mut std::ffi::c_void,
-    ) -> OnEventResult,
-);
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-#[derive(Debug)]
-pub struct RawEvent {
-    pub input: Option<safer_ffi::boxed::Box<Input>>,
-    pub input_closed: Option<safer_ffi::String>,
-    pub stop: bool,
-    pub error: Option<safer_ffi::String>,
-}
-
-#[derive_ReprC]
-#[repr(opaque)]
-#[derive(Debug)]
-pub struct Input {
-    pub id: safer_ffi::String,
-    pub data_array: Option<FFI_ArrowArray>,
-    pub schema: FFI_ArrowSchema,
-    pub metadata: Metadata,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-#[derive(Debug)]
-pub struct Metadata {
-    pub open_telemetry_context: safer_ffi::String,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-pub struct SendOutput {
-    pub send_output: ArcDynFn1<DoraResult, Output>,
-}
-
-#[derive_ReprC]
-#[repr(opaque)]
-#[derive(Debug)]
-pub struct Output {
-    pub id: safer_ffi::String,
-    pub data_array: FFI_ArrowArray,
-    pub schema: FFI_ArrowSchema,
-    pub metadata: Metadata,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[repr(C)]
-#[derive(Debug)]
-pub struct OnEventResult {
-    pub result: DoraResult,
-    pub status: DoraStatus,
-}
-
-#[derive_ReprC]
-#[ffi_export]
-#[derive(Debug)]
-#[repr(u8)]
-pub enum DoraStatus {
-    Continue = 0,
-    Stop = 1,
-    StopAll = 2,
-}
-
-#[ffi_export]
-pub fn dora_read_input_id(input: &Input) -> char_p_boxed {
-    char_p::new(&*input.id)
-}
-
-#[ffi_export]
-pub fn dora_free_input_id(_input_id: char_p_boxed) {}
-
-#[ffi_export]
-pub fn dora_read_data(input: &mut Input) -> Option<safer_ffi::Vec<u8>> {
-    let data_array = input.data_array.take()?;
-    let data = unsafe { arrow::ffi::from_ffi(data_array, &input.schema).ok()? };
-    let array = ArrowData(arrow::array::make_array(data));
-    let bytes: &[u8] = TryFrom::try_from(&array).ok()?;
-    Some(bytes.to_owned().into())
-}
-
-#[ffi_export]
-pub fn dora_free_data(_data: safer_ffi::Vec<u8>) {}
-
-#[ffi_export]
-pub unsafe fn dora_send_operator_output(
-    send_output: &SendOutput,
-    id: safer_ffi::char_p::char_p_ref<'_>,
-    data_ptr: *const u8,
-    data_len: usize,
-) -> DoraResult {
-    let result = || {
-        let data = unsafe { slice::from_raw_parts(data_ptr, data_len) };
-        let arrow_data = data.to_owned().into_arrow();
-        let (data_array, schema) =
-            arrow::ffi::to_ffi(&arrow_data.into_data()).map_err(|err| err.to_string())?;
-        let output = Output {
-            id: id.to_str().to_owned().into(),
-            data_array,
-            schema,
-            metadata: Metadata {
-                open_telemetry_context: String::new().into(), // TODO
-            },
-        };
-        Result::<_, String>::Ok(output)
-    };
-    match result() {
-        Ok(output) => send_output.send_output.call(output),
-        Err(error) => DoraResult {
-            error: Some(Box::new(safer_ffi::String::from(error)).into()),
-        },
-    }
-}
-
-pub fn generate_headers(target_file: &Path) -> ::std::io::Result<()> {
-    ::safer_ffi::headers::builder()
-        .to_file(target_file)?
-        .generate()
-}

ayu-highlight.css

@@ -0,0 +1,79 @@
+/*
+Based off of the Ayu theme
+Original by Dempfi (https://github.com/dempfi/ayu)
+*/
+
+.hljs {
+  display: block;
+  overflow-x: auto;
+  background: #191f26;
+  color: #e6e1cf;
+  padding: 0.5em;
+}
+
+.hljs-comment,
+.hljs-quote {
+  color: #5c6773;
+  font-style: italic;
+}
+
+.hljs-variable,
+.hljs-template-variable,
+.hljs-attribute,
+.hljs-attr,
+.hljs-regexp,
+.hljs-link,
+.hljs-selector-id,
+.hljs-selector-class {
+  color: #ff7733;
+}
+
+.hljs-number,
+.hljs-meta,
+.hljs-builtin-name,
+.hljs-literal,
+.hljs-type,
+.hljs-params {
+  color: #ffee99;
+}
+
+.hljs-string,
+.hljs-bullet {
+  color: #b8cc52;
+}
+
+.hljs-title,
+.hljs-built_in,
+.hljs-section {
+  color: #ffb454;
+}
+
+.hljs-keyword,
+.hljs-selector-tag,
+.hljs-symbol {
+  color: #ff7733;
+}
+
+.hljs-name {
+    color: #36a3d9;
+}
+
+.hljs-tag {
+    color: #00568d;
+}
+
+.hljs-emphasis {
+  font-style: italic;
+}
+
+.hljs-strong {
+  font-weight: bold;
+}
+
+.hljs-addition {
+  color: #91b362;
+}
+
+.hljs-deletion {
+  color: #d96c75;
+}

benches/llms/.gitignore

@@ -1 +0,0 @@
-*.csv

benches/llms/README.md

@@ -1,16 +0,0 @@
-# Benchmark LLM Speed
-
-If you do not have a python virtual environment setup run
-
-'''bash
-uv venv --seed -p 3.11
-'''
-
-Then Use the following command to run the benchmark:
-
-```bash
-dora build transformers.yaml --uv
-dora run transformers.yaml --uv
-```
-
-You should see a benchmark details.

benches/llms/llama_cpp_python.yaml

@@ -1,30 +0,0 @@
-nodes:
-  - id: benchmark_script
-    build: |
-      pip install ../mllm
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-    env:
-      TEXT: "Please only generate the following output: This is a test"
-      TEXT_TRUTH: "This is a test"
-
-  - id: llm
-    build: pip install -e ../../node-hub/dora-llama-cpp-python
-    path: dora-llama-cpp-python
-    inputs:
-      text:
-        source: benchmark_script/text
-        queue-size: 10
-    outputs:
-      - text
-    env:
-      MODEL_NAME_OR_PATH: "Qwen/Qwen2.5-0.5B-Instruct-GGUF"
-      MODEL_FILE_PATTERN: "*fp16.gguf"
-      SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers."
-      MAX_TOKENS: "512"
-      N_GPU_LAYERS: "35" # Enable GPU acceleration
-      N_THREADS: "16" # CPU threads
-      CONTEXT_SIZE: "4096" # Maximum context window

benches/llms/mistralrs.yaml

@@ -1,21 +0,0 @@
-nodes:
-  - id: benchmark_script
-    build: |
-      pip install ../mllm
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-    env:
-      TEXT: "Please only generate the following output: This is a test"
-      TEXT_TRUTH: "This is a test"
-
-  - id: llm
-    build: |
-      cargo build -p dora-mistral-rs --release
-    path: ../../target/release/dora-mistral-rs
-    inputs:
-      text: benchmark_script/text
-    outputs:
-      - text

benches/llms/phi4.yaml

@@ -1,22 +0,0 @@
-nodes:
-  - id: benchmark_script
-    build: |
-      pip install ../mllm
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-    env:
-      TEXT: "Please only generate the following output: This is a test"
-      TEXT_TRUTH: "This is a test"
-
-  - id: llm
-    build: |
-      pip install flash-attn --no-build-isolation
-      pip install -e ../../node-hub/dora-phi4
-    path: dora-phi4
-    inputs:
-      text: benchmark_script/text
-    outputs:
-      - text

benches/llms/qwen2.5.yaml

@@ -1,21 +0,0 @@
-nodes:
-  - id: benchmark_script
-    build: |
-      pip install ../mllm
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-    env:
-      TEXT: "Please only generate the following output: This is a test"
-      TEXT_TRUTH: "This is a test"
-
-  - id: llm
-    build: |
-      pip install -e ../../node-hub/dora-qwen
-    path: dora-qwen
-    inputs:
-      text: benchmark_script/text
-    outputs:
-      - text

benches/llms/transformers.yaml

@@ -1,22 +0,0 @@
-nodes:
-  - id: benchmark_script
-    build: |
-      pip install ../mllm
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-    env:
-      TEXT: "Please only generate the following output: This is a test"
-      TEXT_TRUTH: "This is a test"
-
-  - id: llm
-    build: pip install -e ../../node-hub/dora-transformers
-    path: dora-transformers
-    inputs:
-      text: benchmark_script/text
-    outputs:
-      - text
-    env:
-      MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # Model from Hugging Face

benches/mllm/.gitignore

@@ -1 +0,0 @@
-*.csv

benches/mllm/README.md

@@ -1,10 +0,0 @@
-# Benchmark LLM Speed
-
-Use the following command to run the benchmark:
-
-```bash
-dora build transformers.yaml --uv
-dora run transformers.yaml --uv
-```
-
-You should see a benchmark details.

benches/mllm/benchmark_script.py

@@ -1,220 +0,0 @@
-"""TODO: Add docstring."""
-
-import argparse
-import ast
-
-# Create an empty csv file with header in the current directory if file does not exist
-import csv
-import os
-import time
-from io import BytesIO
-
-import cv2
-import librosa
-import numpy as np
-import pyarrow as pa
-import requests
-from dora import Node
-from PIL import Image
-
-CAT_URL = "https://i.ytimg.com/vi/fzzjgBAaWZw/hqdefault.jpg"
-
-
-def get_cat_image():
-    """
-    Get a cat image as a numpy array.
-
-    :return: Cat image as a numpy array.
-    """
-    # Fetch the image from the URL
-    response = requests.get(CAT_URL)
-    response.raise_for_status()
-
-    # Open the image using PIL
-
-    image = Image.open(BytesIO(response.content))
-    # Convert the image to a numpy array
-
-    image_array = np.array(image)
-    cv2.resize(image_array, (640, 480))
-    # Convert RGB to BGR for
-
-    return image_array
-
-
-AUDIO_URL = "https://github.com/dora-rs/dora-rs.github.io/raw/refs/heads/main/static/Voicy_C3PO%20-Don't%20follow%20me.mp3"
-
-
-def get_c3po_audio():
-    """
-    Download the C-3PO audio and load it into a NumPy array using librosa.
-    """
-    # Download the audio file
-    response = requests.get(AUDIO_URL)
-    if response.status_code != 200:
-        raise Exception(
-            f"Failed to download audio file. Status code: {response.status_code}"
-        )
-
-    # Save the audio file temporarily
-    temp_audio_file = "temp_audio.mp3"
-    with open(temp_audio_file, "wb") as f:
-        f.write(response.content)
-
-    # Load the audio file into a NumPy array using librosa
-    audio_data, sample_rate = librosa.load(temp_audio_file, sr=None)
-
-    # Optionally, you can remove the temporary file after loading
-
-    os.remove(temp_audio_file)
-
-    return audio_data, sample_rate
-
-
-def write_to_csv(filename, header, row):
-    """
-    Create a CSV file with a header if it does not exist, and write a row to it.
-    If the file exists, append the row to the file.
-
-    :param filename: Name of the CSV file.
-    :param header: List of column names to use as the header.
-    :param row: List of data to write as a row in the CSV file.
-    """
-    file_exists = os.path.exists(filename)
-
-    with open(
-        filename, mode="a" if file_exists else "w", newline="", encoding="utf8"
-    ) as file:
-        writer = csv.writer(file)
-
-        # Write the header if the file is being created
-        if not file_exists:
-            writer.writerow(header)
-            print(f"File '{filename}' created with header: {header}")
-
-        # Write the row
-        writer.writerow(row)
-        print(f"Row written to '{filename}': {row}")
-
-
-def main():
-    # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables.
-    """TODO: Add docstring."""
-    parser = argparse.ArgumentParser(description="Simple arrow sender")
-
-    parser.add_argument(
-        "--name",
-        type=str,
-        required=False,
-        help="The name of the node in the dataflow.",
-        default="pyarrow-sender",
-    )
-    parser.add_argument(
-        "--text",
-        type=str,
-        required=False,
-        help="Arrow Data as string.",
-        default=None,
-    )
-
-    args = parser.parse_args()
-
-    text = os.getenv("TEXT", args.text)
-    text_truth = os.getenv("TEXT_TRUTH", args.text)
-
-    cat = get_cat_image()
-    audio, sample_rate = get_c3po_audio()
-    if text is None:
-        raise ValueError(
-            "No data provided. Please specify `TEXT` environment argument or as `--text` argument",
-        )
-    try:
-        text = ast.literal_eval(text)
-    except Exception:  # noqa
-        print("Passing input as string")
-
-    if isinstance(text, (str, int, float)):
-        text = pa.array([text])
-    else:
-        text = pa.array(text)  # initialize pyarrow array
-    node = Node(
-        args.name,
-    )  # provide the name to connect to the dataflow if dynamic node
-    name = node.dataflow_descriptor()["nodes"][1]["path"]
-
-    durations = []
-    speed = []
-    for _ in range(10):
-        node.send_output(
-            "image",
-            pa.array(cat.ravel()),
-            {"encoding": "rgb8", "width": cat.shape[1], "height": cat.shape[0]},
-        )
-        node.send_output(
-            "audio",
-            pa.array(audio.ravel()),
-            {"sample_rate": sample_rate},
-        )
-        time.sleep(0.1)
-        start_time = time.time()
-        node.send_output("text", text)
-        event = node.next()
-        duration = time.time() - start_time
-        if event is not None and event["type"] == "INPUT":
-            received_text = event["value"][0].as_py()
-            tokens = event["metadata"].get("tokens", 6)
-            assert text_truth in received_text, (
-                f"Expected '{text_truth}', got {received_text}"
-            )
-            durations.append(duration)
-            speed.append(tokens / duration)
-            time.sleep(0.1)
-    durations = np.array(durations)
-    speed = np.array(speed)
-    print(
-        f"\nAverage duration: {sum(durations) / len(durations)}"
-        + f"\nMax duration: {max(durations)}"
-        + f"\nMin duration: {min(durations)}"
-        + f"\nMedian duration: {np.median(durations)}"
-        + f"\nMedian frequency: {1 / np.median(durations)}"
-        + f"\nAverage speed: {sum(speed) / len(speed)}"
-        + f"\nMax speed: {max(speed)}"
-        + f"\nMin speed: {min(speed)}"
-        + f"\nMedian speed: {np.median(speed)}"
-        + f"\nTotal tokens: {tokens}"
-    )
-    write_to_csv(
-        "benchmark.csv",
-        [
-            "path",
-            "date",
-            "average_duration(s)",
-            "max_duration(s)",
-            "min_duration(s)",
-            "median_duration(s)",
-            "median_frequency(Hz)",
-            "average_speed(tok/s)",
-            "max_speed(tok/s)",
-            "min_speed(tok/s)",
-            "median_speed(tok/s)",
-            "total_tokens",
-        ],
-        [
-            name,
-            time.strftime("%Y-%m-%d %H:%M:%S"),
-            sum(durations) / len(durations),
-            max(durations),
-            min(durations),
-            np.median(durations),
-            1 / np.median(durations),
-            sum(speed) / len(speed),
-            max(speed),
-            min(speed),
-            np.median(speed),
-            tokens,
-        ],
-    )
-
-
-if __name__ == "__main__":
-    main()

benches/mllm/phi4.yaml

@@ -1,23 +0,0 @@
-nodes:
-  - id: benchmark_script
-    path: benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-      - image
-      - audio
-    env:
-      TEXT: "Please only generate the following output: This is a cat"
-      TEXT_TRUTH: "This is a cat"
-
-  - id: llm
-    build: |
-      pip install -e ../../node-hub/dora-phi4
-    path: dora-phi4
-    inputs:
-      text: benchmark_script/text
-      image: benchmark_script/image
-      audio: benchmark_script/audio
-    outputs:
-      - text

benches/mllm/pyproject.toml

@@ -1,22 +0,0 @@
-[project]
-name = "dora-bench"
-version = "0.1.0"
-description = "Script to benchmark performance of llms while using dora"
-authors = [{ name = "Haixuan Xavier Tao", email = "tao.xavier@outlook.com" }]
-license = { text = "MIT" }
-readme = "README.md"
-requires-python = ">=3.11"
-
-dependencies = [
-    "dora-rs>=0.3.9",
-    "librosa>=0.10.0",
-    "opencv-python>=4.8",
-    "Pillow>=10",
-]
-
-[project.scripts]
-dora-benches = "benchmark_script.main:main"
-
-[build-system]
-requires = ["setuptools>=61", "wheel"]
-build-backend = "setuptools.build_meta"

benches/vlm/.gitignore

@@ -1 +0,0 @@
-*.csv

benches/vlm/README.md

@@ -1,10 +0,0 @@
-# Benchmark LLM Speed
-
-Use the following command to run the benchmark:
-
-```bash
-dora build transformers.yaml --uv
-dora run transformers.yaml --uv
-```
-
-You should see a benchmark details.

benches/vlm/magma.yaml

@@ -1,20 +0,0 @@
-nodes:
-  - id: benchmark_script
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-      - image
-    env:
-      TEXT: "Please only generate the following output: This is a cat"
-      TEXT_TRUTH: "This is a cat"
-
-  - id: llm
-    build: pip install -e ../../node-hub/dora-magma
-    path: dora-magma
-    inputs:
-      text: benchmark_script/text
-      image: benchmark_script/image
-    outputs:
-      - text

benches/vlm/phi4.yaml

@@ -1,22 +0,0 @@
-nodes:
-  - id: benchmark_script
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: llm/text
-    outputs:
-      - text
-      - image
-    env:
-      TEXT: "Please only generate the following output: This is a cat"
-      TEXT_TRUTH: "This is a cat"
-
-  - id: llm
-    build: |
-      pip install flash-attn --no-build-isolation
-      pip install -e ../../node-hub/dora-phi4
-    path: dora-phi4
-    inputs:
-      text: benchmark_script/text
-      image: benchmark_script/image
-    outputs:
-      - text

benches/vlm/qwen2.5vl.yaml

@@ -1,22 +0,0 @@
-nodes:
-  - id: benchmark_script
-    path: ../mllm/benchmark_script.py
-    inputs:
-      text: vlm/text
-    outputs:
-      - text
-      - image
-    env:
-      TEXT: "Please only generate the following output: This is a cat"
-      TEXT_TRUTH: "This is a cat"
-
-  - id: vlm
-    # Comment flash_attn if not on cuda hardware
-    build: |
-      pip install -e ../../node-hub/dora-qwen2-5-vl
-    path: dora-qwen2-5-vl
-    inputs:
-      image: benchmark_script/image
-      text: benchmark_script/text
-    outputs:
-      - text