Merge branch 'main' into improve-nix-environment

.github/workflows/ci.yml

@@ -118,6 +118,9 @@ jobs:
       - name: "Rust Dataflow example"
         timeout-minutes: 30
         run: cargo run --example rust-dataflow
+      - name: "Rust Git Dataflow example"
+        timeout-minutes: 30
+        run: cargo run --example rust-dataflow-git
       - name: "Multiple Daemons example"
         timeout-minutes: 30
         run: cargo run --example multiple-daemons
@@ -194,7 +197,7 @@ jobs:
           required-ros-distributions: humble
       - run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"'
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           enable-cache: true
       - name: Install pyarrow
@@ -209,11 +212,11 @@ jobs:
           source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node &
           source /opt/ros/humble/setup.bash && ros2 run examples_rclcpp_minimal_service service_main &
           cargo run --example rust-ros2-dataflow --features="ros2-examples"
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         if: runner.os != 'Windows'
         with:
           python-version: "3.8"
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         if: runner.os == 'Windows'
         with:
           python-version: "3.10"
@@ -321,7 +324,7 @@ jobs:
           dora stop --name ci-rust-dynamic  --grace-duration 5s
           dora destroy
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           # TODO: Support Python 3.13 when https://github.com/pytorch/pytorch/issues/130249 is fixed
           python-version: "3.12"
@@ -339,35 +342,42 @@ jobs:
           # Test Python template Project
           dora new test_python_project --lang python --internal-create-with-path-dependencies
           cd test_python_project
-          uv venv --seed -p 3.11
+          uv venv --seed -p 3.12
           uv pip install -e ../apis/python/node
-          dora build dataflow.yml --uv
           uv pip install ruff pytest
 
+          echo "Running dora up"
+          dora up
+          echo "Running dora build"
+          dora build dataflow.yml --uv
+
           # Check Compliancy
           uv run ruff check .
           uv run pytest
 
           export OPERATING_MODE=SAVE
-          dora up
+          echo "Running dora list"
           dora list
           dora build dataflow.yml --uv
           echo "Running CI Python Test"
           dora start dataflow.yml --name ci-python-test --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-test  --grace-duration 5s
           dora destroy
+          sleep 5
 
           cd ..
 
           # Run Python Node Example
           echo "Running Python Node Example"
           dora up
-          uv venv --seed -p 3.11
+          uv venv --seed -p 3.12
           uv pip install -e apis/python/node
           dora build examples/python-dataflow/dataflow.yml --uv
           dora start examples/python-dataflow/dataflow.yml --name ci-python --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python  --grace-duration 30s
 
           # Run Python Dynamic Node Example
@@ -376,15 +386,18 @@ jobs:
           dora start examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach --uv
           uv run opencv-plot --name plot
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-dynamic  --grace-duration 30s
 
           # Run Python Operator Example
           echo "Running CI Operator Test"
           dora start examples/python-operator-dataflow/dataflow.yml --name ci-python-operator --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-operator  --grace-duration 30s
 
           dora destroy
+          sleep 5
 
           # Run Python queue latency test
           echo "Running CI Queue Latency Test"

.github/workflows/node_hub_test.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -euo
 
+# Check if we are running in a GitHub Actions environment
+CI=${GITHUB_ACTIONS:-false}
+
 # List of ignored modules 
 ignored_folders=("dora-parler" "dora-opus" "dora-internvl" "dora-magma")
 
@@ -13,6 +16,32 @@ dir=$(pwd)
 # Get the base name of the directory (without the path)
 base_dir=$(basename "$dir")
 
+# Large node list requiring space cleanup
+large_node=("dora-phi4")
+
+export PYTEST_ADDOPTS="-x"
+
+# Check if the current directory is in the large node list and if we're in the CI environment
+if [[ " ${large_node[@]} " =~ " ${base_dir} " ]] && [[ "$CI" == "true" ]]; then
+    echo "Running cleanup for $base_dir..."
+    sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+    # 1.4GB
+    sudo rm -rf /opt/hostedtoolcache/go || :
+    # 489MB
+    sudo rm -rf /opt/hostedtoolcache/PyPy || :
+    # 376MB
+    sudo rm -rf /opt/hostedtoolcache/node || :
+    # Remove Web browser packages
+    sudo apt purge -y \
+    firefox \
+    google-chrome-stable \
+    microsoft-edge-stable
+    sudo rm -rf /usr/local/lib/android/ 
+    sudo rm -rf /usr/share/dotnet/
+    sudo rm -rf /opt/ghc/
+fi
+
+
 # Check if the directory name is in the ignored list
 if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
     echo "Skipping $base_dir as we cannot test it on the CI..."
@@ -69,7 +98,7 @@ else
             maturin publish --target x86_64-apple-darwin --skip-existing --zig
         fi
 
-    elif [[ "$(uname)" = "Linux" ]]; then
+    elif [[ "$(uname)" = "Linux" ]] || [[ "$CI" == "false" ]]; then
         if [ -f "$dir/Cargo.toml" ]; then
             echo "Running build and tests for Rust project in $dir..."
             cargo check
@@ -94,7 +123,7 @@ else
             else
                 uv run pytest
             fi
-            if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
+            if [ "${GITHUB_EVENT_NAME:-false}" == "release" ] || [ "${GITHUB_EVENT_NAME:-false}" == "workflow_dispatch" ]; then
                 uv build
                 uv publish --check-url https://pypi.org/simple
             fi

.github/workflows/pip-release.yml

@@ -66,6 +66,7 @@ jobs:
           args: --release --out dist --zig
           manylinux: manylinux_2_28
           working-directory: ${{ matrix.repository.path }}
+          before-script-linux: sudo apt-get install libatomic1-i386-cross libatomic1-armhf-cross && mkdir -p $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/ && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so.1 && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 /opt/hostedtoolcache/Python/3.8.18/x64/lib/libatomic.so.1 && mkdir -p $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/ && ln -s /usr/arm-linux-gnueabihf/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/libatomic.so
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4

.gitignore

@@ -35,7 +35,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-build/
+/build/
 develop-eggs/
 dist/
 downloads/

Cargo.toml

@@ -50,36 +50,37 @@ members = [
 [workspace.package]
 edition = "2021"
 # Make sure to also bump `apis/node/python/__init__.py` version.
-version = "0.3.11"
+version = "0.3.12"
 description = "`dora` goal is to be a low latency, composable, and distributed data flow."
 documentation = "https://dora-rs.ai"
 license = "Apache-2.0"
 repository = "https://github.com/dora-rs/dora/"
 
 [workspace.dependencies]
-dora-node-api = { version = "0.3.11", path = "apis/rust/node", default-features = false }
-dora-node-api-python = { version = "0.3.11", path = "apis/python/node", default-features = false }
-dora-operator-api = { version = "0.3.11", path = "apis/rust/operator", default-features = false }
-dora-operator-api-macros = { version = "0.3.11", path = "apis/rust/operator/macros" }
-dora-operator-api-types = { version = "0.3.11", path = "apis/rust/operator/types" }
-dora-operator-api-python = { version = "0.3.11", path = "apis/python/operator" }
-dora-operator-api-c = { version = "0.3.11", path = "apis/c/operator" }
-dora-node-api-c = { version = "0.3.11", path = "apis/c/node" }
-dora-core = { version = "0.3.11", path = "libraries/core" }
-dora-arrow-convert = { version = "0.3.11", path = "libraries/arrow-convert" }
-dora-tracing = { version = "0.3.11", path = "libraries/extensions/telemetry/tracing" }
-dora-metrics = { version = "0.3.11", path = "libraries/extensions/telemetry/metrics" }
-dora-download = { version = "0.3.11", path = "libraries/extensions/download" }
-shared-memory-server = { version = "0.3.11", path = "libraries/shared-memory-server" }
-communication-layer-request-reply = { version = "0.3.11", path = "libraries/communication-layer/request-reply" }
-dora-runtime = { version = "0.3.11", path = "binaries/runtime" }
-dora-daemon = { version = "0.3.11", path = "binaries/daemon" }
-dora-coordinator = { version = "0.3.11", path = "binaries/coordinator" }
-dora-ros2-bridge = { version = "0.3.11", path = "libraries/extensions/ros2-bridge" }
-dora-ros2-bridge-msg-gen = { version = "0.3.11", path = "libraries/extensions/ros2-bridge/msg-gen" }
+dora-node-api = { version = "0.3.12", path = "apis/rust/node", default-features = false }
+dora-node-api-python = { version = "0.3.12", path = "apis/python/node", default-features = false }
+dora-operator-api = { version = "0.3.12", path = "apis/rust/operator", default-features = false }
+dora-operator-api-macros = { version = "0.3.12", path = "apis/rust/operator/macros" }
+dora-operator-api-types = { version = "0.3.12", path = "apis/rust/operator/types" }
+dora-operator-api-python = { version = "0.3.12", path = "apis/python/operator" }
+dora-operator-api-c = { version = "0.3.12", path = "apis/c/operator" }
+dora-node-api-c = { version = "0.3.12", path = "apis/c/node" }
+dora-core = { version = "0.3.12", path = "libraries/core" }
+dora-arrow-convert = { version = "0.3.12", path = "libraries/arrow-convert" }
+dora-tracing = { version = "0.3.12", path = "libraries/extensions/telemetry/tracing" }
+dora-metrics = { version = "0.3.12", path = "libraries/extensions/telemetry/metrics" }
+dora-download = { version = "0.3.12", path = "libraries/extensions/download" }
+shared-memory-server = { version = "0.3.12", path = "libraries/shared-memory-server" }
+communication-layer-request-reply = { version = "0.3.12", path = "libraries/communication-layer/request-reply" }
+dora-cli = { version = "0.3.12", path = "binaries/cli" }
+dora-runtime = { version = "0.3.12", path = "binaries/runtime" }
+dora-daemon = { version = "0.3.12", path = "binaries/daemon" }
+dora-coordinator = { version = "0.3.12", path = "binaries/coordinator" }
+dora-ros2-bridge = { version = "0.3.12", path = "libraries/extensions/ros2-bridge" }
+dora-ros2-bridge-msg-gen = { version = "0.3.12", path = "libraries/extensions/ros2-bridge/msg-gen" }
 dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" }
 # versioned independently from the other dora crates
-dora-message = { version = "0.4.4", path = "libraries/message" }
+dora-message = { version = "0.5.0", path = "libraries/message" }
 arrow = { version = "54.2.1" }
 arrow-schema = { version = "54.2.1" }
 arrow-data = { version = "54.2.1" }
@@ -91,6 +92,8 @@ pyo3 = { version = "0.23", features = [
     "multiple-pymethods",
 ] }
 pythonize = "0.23"
+git2 = { version = "0.18.0", features = ["vendored-openssl"] }
+serde_yaml = "0.9.33"
 
 [package]
 name = "dora-examples"
@@ -107,6 +110,7 @@ ros2-examples = []
 [dev-dependencies]
 eyre = "0.6.8"
 tokio = "1.24.2"
+dora-cli = { workspace = true }
 dora-coordinator = { workspace = true }
 dora-core = { workspace = true }
 dora-message = { workspace = true }
@@ -135,6 +139,10 @@ path = "examples/vlm/run.rs"
 name = "rust-dataflow"
 path = "examples/rust-dataflow/run.rs"
 
+[[example]]
+name = "rust-dataflow-git"
+path = "examples/rust-dataflow-git/run.rs"
+
 [[example]]
 name = "rust-ros2-dataflow"
 path = "examples/rust-ros2-dataflow/run.rs"

Changelog.md

@@ -1,5 +1,59 @@
 # Changelog
 
+## v0.3.12 (2025-06-30)
+
+## What's Changed
+
+- Implemented dora-cotracker node by @ShashwatPatil in https://github.com/dora-rs/dora/pull/931
+- Minor fix and add boxes2d example to facebook/cotracker by @haixuanTao in https://github.com/dora-rs/dora/pull/950
+- Update Rust crate tokio to v1.44.2 [SECURITY] by @renovate in https://github.com/dora-rs/dora/pull/951
+- Post 3.11 release fix by @haixuanTao in https://github.com/dora-rs/dora/pull/954
+- Bump crossbeam-channel from 0.5.14 to 0.5.15 by @dependabot in https://github.com/dora-rs/dora/pull/959
+- Added E ruff flag for pydocstyle by @7SOMAY in https://github.com/dora-rs/dora/pull/958
+- Revert "Added E ruff flag for better code quality [skip ci]" by @haixuanTao in https://github.com/dora-rs/dora/pull/968
+- Ease of use changes in benches for issue #957 by @Ignavar in https://github.com/dora-rs/dora/pull/969
+- Reachy cotracker by @haixuanTao in https://github.com/dora-rs/dora/pull/972
+- Improve rav1e by @haixuanTao in https://github.com/dora-rs/dora/pull/974
+- Fix pyrealsense by @haixuanTao in https://github.com/dora-rs/dora/pull/973
+- Added Self Uninstall Command by @Shar-jeel-Sajid in https://github.com/dora-rs/dora/pull/944
+- Improve benchmark implementation & Add warning for discarding events by @Mivik in https://github.com/dora-rs/dora/pull/971
+- docs: Updated README: Added comprehensive usage documentation with vi… by @LeonRust in https://github.com/dora-rs/dora/pull/983
+- Fix rerun-viewer example. by @francocipollone in https://github.com/dora-rs/dora/pull/989
+- docs: add license badge by @Radovenchyk in https://github.com/dora-rs/dora/pull/996
+- Disable sccache for `musllinux` jobs by @haixuanTao in https://github.com/dora-rs/dora/pull/1000
+- Remove unused sysinfo monitor by @Mivik in https://github.com/dora-rs/dora/pull/1007
+- Refactor Python CUDA IPC API by @Mivik in https://github.com/dora-rs/dora/pull/1002
+- fix terminal not printing stdout on nvml warning by @haixuanTao in https://github.com/dora-rs/dora/pull/1008
+- Fix issue #1006: [Brief description of the fix] by @sohamukute in https://github.com/dora-rs/dora/pull/1013
+- Improving so100 usability by @haixuanTao in https://github.com/dora-rs/dora/pull/988
+- Add dora-mediapipe node for quick human pose estimation by @haixuanTao in https://github.com/dora-rs/dora/pull/986
+- Bump torch to 2.7 by @haixuanTao in https://github.com/dora-rs/dora/pull/1015
+- refactor(tracing): use builder style by @sjfhsjfh in https://github.com/dora-rs/dora/pull/1009
+- Fix spawning runtime through python when it is installed with pip by @haixuanTao in https://github.com/dora-rs/dora/pull/1011
+- chore(deps): update dependency numpy to v2 by @renovate in https://github.com/dora-rs/dora/pull/1014
+- Fix error when multiple visualization key is active and when urdf_transform env variable is not present by @haixuanTao in https://github.com/dora-rs/dora/pull/1016
+- Update pyrealsense2 Dependencies for L515 Support and Fix README wget Link by @kingchou007 in https://github.com/dora-rs/dora/pull/1021
+- Minor fix for mujoco sim by @haixuanTao in https://github.com/dora-rs/dora/pull/1023
+- dora-mujoco simulation node with example for controlling any arm by @ShashwatPatil in https://github.com/dora-rs/dora/pull/1012
+- fix ros CI/CD by @haixuanTao in https://github.com/dora-rs/dora/pull/1027
+- dora-vggt by @haixuanTao in https://github.com/dora-rs/dora/pull/1024
+- Adding vision to openai server by @haixuanTao in https://github.com/dora-rs/dora/pull/1025
+- Revert "Adding vision to openai server" by @haixuanTao in https://github.com/dora-rs/dora/pull/1031
+- Expose AllInputClosed message as a Stop message by @haixuanTao in https://github.com/dora-rs/dora/pull/1026
+- Add support for git repository sources for nodes by @phil-opp in https://github.com/dora-rs/dora/pull/901
+- Adding vision to rust openai proxy server by @haixuanTao in https://github.com/dora-rs/dora/pull/1033
+- Add automatic robot descriptions URDF retrieval from https://github.com/robot-descriptions/robot_descriptions.py by @haixuanTao in https://github.com/dora-rs/dora/pull/1032
+
+## New Contributors
+
+- @Mivik made their first contribution in https://github.com/dora-rs/dora/pull/971
+- @francocipollone made their first contribution in https://github.com/dora-rs/dora/pull/989
+- @sohamukute made their first contribution in https://github.com/dora-rs/dora/pull/1013
+- @sjfhsjfh made their first contribution in https://github.com/dora-rs/dora/pull/1009
+- @kingchou007 made their first contribution in https://github.com/dora-rs/dora/pull/1021
+
+**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.11...v0.3.12
+
 ## v0.3.11 (2025-04-07)
 
 ## What's Changed

README.md

@@ -62,6 +62,8 @@
 <details open>
 <summary><b>2025</b></summary>
 
+- \[07/25\] Added Kornia rust nodes in the hub for V4L / Gstreamer cameras and Sobel image processing.
+- \[06/25\] Add support for git based node, dora-vggt for multi-camera depth estimation, and adding robot_descriptions_py as a default way to get urdfs within dora.
 - \[05/25\] Add support for dora-pytorch-kinematics for fk and ik, dora-mediapipe for pose estimation, dora-rustypot for rust serialport read/write, points2d and points3d visualization in rerun.
 - \[04/25\] Add support for dora-cotracker to track any point on a frame, dora-rav1e AV1 encoding up to 12bit and dora-dav1d AV1 decoding,
 - \[03/25\] Add support for dora async Python.
@@ -102,6 +104,8 @@
 | Camera                        | [PyOrbbeckSDK](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyorbbecksdk)                | 📐                  | Image and depth from Orbbeck Camera              | ![Downloads](https://img.shields.io/pypi/dm/dora-pyorbbecksdk?label=%20)      | ![License](https://img.shields.io/pypi/l/dora-pyorbbecksdk?label=%20)      |
 | Camera                        | [PyRealsense](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyrealsense)                  | Linux🆗 <br> Mac🛠️  | Image and depth from Realsense                   | ![Downloads](https://img.shields.io/pypi/dm/dora-pyrealsense?label=%20)       | ![License](https://img.shields.io/pypi/l/dora-pyrealsense?label=%20)       |
 | Camera                        | [OpenCV Video Capture](https://github.com/dora-rs/dora/blob/main/node-hub/opencv-video-capture)     | ✅                  | Image stream from OpenCV Camera                  | ![Downloads](https://img.shields.io/pypi/dm/opencv-video-capture?label=%20)   | ![License](https://img.shields.io/pypi/l/opencv-video-capture?label=%20)   |
+| Camera                        | [Kornia V4L Capture](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-v4l-capture)         | ✅                  | Video stream for Linux Camera (rust)             |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)   |
+| Camera                        | [Kornia GST Capture](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-gst-capture)         | ✅                  | Video Capture using Gstreamer (rust)             |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)   |
 | Peripheral                    | [Keyboard](https://github.com/dora-rs/dora/blob/main/node-hub/dora-keyboard)                        | ✅                  | Keyboard char listener                           | ![Downloads](https://img.shields.io/pypi/dm/dora-keyboard?label=%20)          | ![License](https://img.shields.io/pypi/l/dora-keyboard?label=%20)          |
 | Peripheral                    | [Microphone](https://github.com/dora-rs/dora/blob/main/node-hub/dora-microphone)                    | ✅                  | Audio from microphone                            | ![Downloads](https://img.shields.io/pypi/dm/dora-microphone?label=%20)        | ![License](https://img.shields.io/pypi/l/dora-microphone?label=%20)        |
 | Peripheral                    | [PyAudio(Speaker)](https://github.com/dora-rs/dora/blob/main/node-hub/dora-pyaudio)                 | ✅                  | Output audio from speaker                        | ![Downloads](https://img.shields.io/pypi/dm/dora-pyaudio?label=%20)           | ![License](https://img.shields.io/pypi/l/dora-pyaudio?label=%20)           |
@@ -134,6 +138,7 @@
 | Simulator                     | [Mujoco](https://github.com/dora-rs/dora-lerobot/blob/main/node-hub/mujoco-client)                  | 📐                  | Mujoco Simulator                                 |                                                                               |                                                                            |
 | Simulator                     | [Carla](https://github.com/dora-rs/dora-drives)                                                     | 📐                  | Carla Simulator                                  |                                                                               |                                                                            |
 | Simulator                     | [Gymnasium](https://github.com/dora-rs/dora-lerobot/blob/main/gym_dora)                             | 📐                  | Experimental OpenAI Gymnasium bridge             |                                                                               |                                                                            |
+| Image Processing              | [Kornia Sobel Operator](https://github.com/kornia/dora-nodes-hub/tree/main/kornia-imgproc-sobel)   | ✅                  | Kornia image processing Sobel operator (rust)  |                                                                               | ![License](https://img.shields.io/badge/license-Apache%202-blue)   |
 
 ## Examples
 
@@ -144,6 +149,7 @@
 | Vision         | [Vision Language Model(VLM)](https://github.com/dora-rs/dora/blob/main/examples/vlm)                         | Use a VLM to understand images.                     | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fvlm&label=%20)                   |
 | Vision         | [YOLO](https://github.com/dora-rs/dora/blob/main/examples/python-dataflow)                                   | Use YOLO to detect object within image.             | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpython-dataflow&label=%20)       |
 | Vision         | [Camera](https://github.com/dora-rs/dora/blob/main/examples/camera)                                          | Simple webcam plot example                          | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fcamera&label=%20)                |
+| Vision         | [Image Processing](https://github.com/kornia/kornia-rs/tree/main/examples/dora)                              | Multi camera image processing                       |                                                                                                                    |
 | Model Training | [Piper RDT](https://github.com/dora-rs/dora/blob/main/examples/piper)                                        | Piper RDT Pipeline                                  | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fpiper&label=%20)                 |
 | Model Training | [LeRobot - Alexander Koch](https://raw.githubusercontent.com/dora-rs/dora-lerobot/refs/heads/main/README.md) | Training Alexander Koch Low Cost Robot with LeRobot | ![License](https://img.shields.io/github/last-commit/dora-rs/dora-lerobot?path=robots&label=%20)                   |
 | ROS2           | [C++ ROS2 Example](https://github.com/dora-rs/dora/blob/main/examples/c++-ros2-dataflow)                     | Example using C++ ROS2                              | ![License](https://img.shields.io/github/last-commit/dora-rs/dora?path=examples%2Fc%2b%2b-ros2-dataflow&label=%20) |

apis/c++/node/src/lib.rs

@@ -144,7 +144,7 @@ pub struct DoraEvent(Option<Event>);
 fn event_type(event: &DoraEvent) -> ffi::DoraEventType {
     match &event.0 {
         Some(event) => match event {
-            Event::Stop => ffi::DoraEventType::Stop,
+            Event::Stop(_) => ffi::DoraEventType::Stop,
             Event::Input { .. } => ffi::DoraEventType::Input,
             Event::InputClosed { .. } => ffi::DoraEventType::InputClosed,
             Event::Error(_) => ffi::DoraEventType::Error,

apis/c/node/src/lib.rs

@@ -91,7 +91,7 @@ pub unsafe extern "C" fn dora_next_event(context: *mut c_void) -> *mut c_void {
 pub unsafe extern "C" fn read_dora_event_type(event: *const ()) -> EventType {
     let event: &Event = unsafe { &*event.cast() };
     match event {
-        Event::Stop => EventType::Stop,
+        Event::Stop(_) => EventType::Stop,
         Event::Input { .. } => EventType::Input,
         Event::InputClosed { .. } => EventType::InputClosed,
         Event::Error(_) => EventType::Error,

apis/python/node/Cargo.toml

@@ -21,10 +21,10 @@ dora-node-api = { workspace = true }
 dora-operator-api-python = { workspace = true }
 pyo3.workspace = true
 eyre = "0.6"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 flume = "0.10.14"
 dora-runtime = { workspace = true, features = ["tracing", "metrics", "python"] }
-dora-daemon = { workspace = true }
+dora-cli = { workspace = true }
 dora-download = { workspace = true }
 arrow = { workspace = true, features = ["pyarrow"] }
 pythonize = { workspace = true }
@@ -33,6 +33,9 @@ dora-ros2-bridge-python = { workspace = true }
 # pyo3_special_method_derive = "0.4.2"
 tokio = { version = "1.24.2", features = ["rt"] }
 
+[build-dependencies]
+pyo3-build-config = "0.23"
+
 [lib]
 name = "dora"
 crate-type = ["cdylib"]

apis/python/node/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    pyo3_build_config::add_extension_module_link_args();
+}

apis/python/node/pyproject.toml

@@ -22,3 +22,11 @@ extend-select = [
     "D",  # pydocstyle
     "UP",
 ]
+
+[tool.maturin.target.x86_64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"
+
+[tool.maturin.target.aarch64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"

apis/python/node/src/lib.rs

@@ -6,7 +6,6 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use arrow::pyarrow::{FromPyArrow, ToPyArrow};
-use dora_daemon::Daemon;
 use dora_download::download_file;
 use dora_node_api::dora_core::config::NodeId;
 use dora_node_api::dora_core::descriptor::source_is_url;
@@ -231,7 +230,7 @@ impl Node {
     /// :rtype: dict
     pub fn dataflow_descriptor(&mut self, py: Python) -> eyre::Result<PyObject> {
         Ok(
-            pythonize::pythonize(py, &self.node.get_mut().dataflow_descriptor())
+            pythonize::pythonize(py, &self.node.get_mut().dataflow_descriptor()?)
                 .map(|x| x.unbind())?,
         )
     }
@@ -382,19 +381,7 @@ pub fn resolve_dataflow(dataflow: String) -> eyre::Result<PathBuf> {
 #[pyfunction]
 #[pyo3(signature = (dataflow_path, uv=None))]
 pub fn run(dataflow_path: String, uv: Option<bool>) -> eyre::Result<()> {
-    let dataflow_path = resolve_dataflow(dataflow_path).context("could not resolve dataflow")?;
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .context("tokio runtime failed")?;
-    let result = rt.block_on(Daemon::run_dataflow(&dataflow_path, uv.unwrap_or_default()))?;
-    match result.is_ok() {
-        true => Ok(()),
-        false => Err(eyre::eyre!(
-            "Dataflow failed to run with error: {:?}",
-            result.node_results
-        )),
-    }
+    dora_cli::command::run(dataflow_path, uv.unwrap_or_default())
 }
 
 #[pymodule]

apis/python/operator/Cargo.toml

@@ -14,7 +14,7 @@ repository.workspace = true
 dora-node-api = { workspace = true }
 pyo3 = { workspace = true, features = ["eyre", "abi3-py37"] }
 eyre = "0.6"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 flume = "0.10.14"
 arrow = { workspace = true, features = ["pyarrow"] }
 arrow-schema = { workspace = true }

apis/python/operator/src/lib.rs

@@ -6,7 +6,7 @@ use std::{
 use arrow::pyarrow::ToPyArrow;
 use dora_node_api::{
     merged::{MergeExternalSend, MergedEvent},
-    DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter,
+    DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, StopCause,
 };
 use eyre::{Context, Result};
 use futures::{Stream, StreamExt};
@@ -146,7 +146,7 @@ impl PyEvent {
 
     fn ty(event: &Event) -> &str {
         match event {
-            Event::Stop => "STOP",
+            Event::Stop(_) => "STOP",
             Event::Input { .. } => "INPUT",
             Event::InputClosed { .. } => "INPUT_CLOSED",
             Event::Error(_) => "ERROR",
@@ -158,6 +158,11 @@ impl PyEvent {
         match event {
             Event::Input { id, .. } => Some(id),
             Event::InputClosed { id } => Some(id),
+            Event::Stop(cause) => match cause {
+                StopCause::Manual => Some("MANUAL"),
+                StopCause::AllInputsClosed => Some("ALL_INPUTS_CLOSED"),
+                &_ => None,
+            },
             _ => None,
         }
     }

apis/rust/node/Cargo.toml

@@ -17,7 +17,7 @@ dora-core = { workspace = true }
 dora-message = { workspace = true }
 shared-memory-server = { workspace = true }
 eyre = "0.6.7"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 tracing = "0.1.33"
 flume = "0.10.14"
 bincode = "1.3.3"

apis/rust/node/src/event_stream/event.rs

@@ -10,7 +10,7 @@ use shared_memory_extended::{Shmem, ShmemConf};
 #[derive(Debug)]
 #[non_exhaustive]
 pub enum Event {
-    Stop,
+    Stop(StopCause),
     Reload {
         operator_id: Option<OperatorId>,
     },
@@ -25,6 +25,13 @@ pub enum Event {
     Error(String),
 }
 
+#[derive(Debug, Clone)]
+#[non_exhaustive]
+pub enum StopCause {
+    Manual,
+    AllInputsClosed,
+}
+
 pub enum RawData {
     Empty,
     Vec(AVec<u8, ConstAlign<128>>),

apis/rust/node/src/event_stream/mod.rs

@@ -11,7 +11,7 @@ use dora_message::{
     node_to_daemon::{DaemonRequest, Timestamped},
     DataflowId,
 };
-pub use event::{Event, MappedInputData, RawData};
+pub use event::{Event, MappedInputData, RawData, StopCause};
 use futures::{
     future::{select, Either},
     Stream, StreamExt,
@@ -199,7 +199,7 @@ impl EventStream {
     fn convert_event_item(item: EventItem) -> Event {
         match item {
             EventItem::NodeEvent { event, ack_channel } => match event {
-                NodeEvent::Stop => Event::Stop,
+                NodeEvent::Stop => Event::Stop(event::StopCause::Manual),
                 NodeEvent::Reload { operator_id } => Event::Reload { operator_id },
                 NodeEvent::InputClosed { id } => Event::InputClosed { id },
                 NodeEvent::Input { id, metadata, data } => {
@@ -234,13 +234,7 @@ impl EventStream {
                         Err(err) => Event::Error(format!("{err:?}")),
                     }
                 }
-                NodeEvent::AllInputsClosed => {
-                    let err = eyre!(
-                        "received `AllInputsClosed` event, which should be handled by background task"
-                    );
-                    tracing::error!("{err:?}");
-                    Event::Error(err.wrap_err("internal error").to_string())
-                }
+                NodeEvent::AllInputsClosed => Event::Stop(event::StopCause::AllInputsClosed),
             },
 
             EventItem::FatalError(err) => {

apis/rust/node/src/event_stream/thread.rs

@@ -92,6 +92,7 @@ fn event_stream_loop(
     clock: Arc<uhlc::HLC>,
 ) {
     let mut tx = Some(tx);
+    let mut close_tx = false;
     let mut pending_drop_tokens: Vec<(DropToken, flume::Receiver<()>, Instant, u64)> = Vec::new();
     let mut drop_tokens = Vec::new();
 
@@ -135,10 +136,8 @@ fn event_stream_loop(
                     data: Some(data), ..
                 } => data.drop_token(),
                 NodeEvent::AllInputsClosed => {
-                    // close the event stream
-                    tx = None;
-                    // skip this internal event
-                    continue;
+                    close_tx = true;
+                    None
                 }
                 _ => None,
             };
@@ -166,6 +165,10 @@ fn event_stream_loop(
             } else {
                 tracing::warn!("dropping event because event `tx` was already closed: `{inner:?}`");
             }
+
+            if close_tx {
+                tx = None;
+            };
         }
     };
     if let Err(err) = result {

apis/rust/node/src/lib.rs

@@ -20,7 +20,7 @@ pub use dora_message::{
     metadata::{Metadata, MetadataParameters, Parameter},
     DataflowId,
 };
-pub use event_stream::{merged, Event, EventStream, MappedInputData, RawData};
+pub use event_stream::{merged, Event, EventStream, MappedInputData, RawData, StopCause};
 pub use flume::Receiver;
 pub use node::{arrow_utils, DataSample, DoraNode, ZERO_COPY_THRESHOLD};
 

apis/rust/node/src/node/arrow_utils.rs

@@ -11,7 +11,7 @@ fn required_data_size_inner(array: &ArrayData, next_offset: &mut usize) {
     for (buffer, spec) in array.buffers().iter().zip(&layout.buffers) {
         // consider alignment padding
         if let BufferSpec::FixedWidth { alignment, .. } = spec {
-            *next_offset = (*next_offset + alignment - 1) / alignment * alignment;
+            *next_offset = (*next_offset).div_ceil(*alignment) * alignment;
         }
         *next_offset += buffer.len();
     }
@@ -42,7 +42,7 @@ fn copy_array_into_sample_inner(
         );
         // add alignment padding
         if let BufferSpec::FixedWidth { alignment, .. } = spec {
-            *next_offset = (*next_offset + alignment - 1) / alignment * alignment;
+            *next_offset = (*next_offset).div_ceil(*alignment) * alignment;
         }
 
         target_buffer[*next_offset..][..len].copy_from_slice(buffer.as_slice());

apis/rust/node/src/node/mod.rs

@@ -44,6 +44,7 @@ mod drop_stream;
 
 pub const ZERO_COPY_THRESHOLD: usize = 4096;
 
+#[allow(dead_code)]
 enum TokioRuntime {
     Runtime(Runtime),
     Handle(Handle),
@@ -60,7 +61,7 @@ pub struct DoraNode {
     drop_stream: DropStream,
     cache: VecDeque<ShmemHandle>,
 
-    dataflow_descriptor: Descriptor,
+    dataflow_descriptor: serde_yaml::Result<Descriptor>,
     warned_unknown_output: BTreeSet<DataId>,
     _rt: TokioRuntime,
 }
@@ -158,10 +159,9 @@ impl DoraNode {
             ),
         };
 
-        let id = format!("{}/{}", dataflow_id, node_id);
-
         #[cfg(feature = "metrics")]
         {
+            let id = format!("{}/{}", dataflow_id, node_id);
             let monitor_task = async move {
                 if let Err(e) = run_metrics_monitor(id.clone())
                     .await
@@ -200,7 +200,7 @@ impl DoraNode {
             sent_out_shared_memory: HashMap::new(),
             drop_stream,
             cache: VecDeque::new(),
-            dataflow_descriptor,
+            dataflow_descriptor: serde_yaml::from_value(dataflow_descriptor),
             warned_unknown_output: BTreeSet::new(),
             _rt: rt,
         };
@@ -449,8 +449,15 @@ impl DoraNode {
     /// Returns the full dataflow descriptor that this node is part of.
     ///
     /// This method returns the parsed dataflow YAML file.
-    pub fn dataflow_descriptor(&self) -> &Descriptor {
-        &self.dataflow_descriptor
+    pub fn dataflow_descriptor(&self) -> eyre::Result<&Descriptor> {
+        match &self.dataflow_descriptor {
+            Ok(d) => Ok(d),
+            Err(err) => eyre::bail!(
+                "failed to parse dataflow descriptor: {err}\n\n
+                This might be caused by mismatched version numbers of dora \
+                daemon and the dora node API"
+            ),
+        }
     }
 }
 

binaries/cli/Cargo.toml

@@ -27,7 +27,7 @@ dora-node-api-c = { workspace = true }
 dora-operator-api-c = { workspace = true }
 dora-download = { workspace = true }
 serde = { version = "1.0.136", features = ["derive"] }
-serde_yaml = "0.9.11"
+serde_yaml = { workspace = true }
 webbrowser = "0.8.3"
 serde_json = "1.0.86"
 termcolor = "1.1.3"
@@ -37,6 +37,7 @@ communication-layer-request-reply = { workspace = true }
 notify = "5.1.0"
 ctrlc = "3.2.5"
 tracing = "0.1.36"
+tracing-log = "0.2.0"
 dora-tracing = { workspace = true, optional = true }
 bat = "0.24.0"
 dora-daemon = { workspace = true }
@@ -50,7 +51,7 @@ tabwriter = "1.4.0"
 log = { version = "0.4.21", features = ["serde"] }
 colored = "2.1.0"
 env_logger = "0.11.3"
-self_update = { version = "0.27.0", features = [
+self_update = { version = "0.42.0", features = [
     "rustls",
     "archive-zip",
     "archive-tar",
@@ -61,7 +62,11 @@ pyo3 = { workspace = true, features = [
     "abi3",
 ], optional = true }
 self-replace = "1.5.0"
+dunce = "1.0.5"
+git2 = { workspace = true }
 
+[build-dependencies]
+pyo3-build-config = "0.23"
 
 [lib]
 name = "dora_cli"

binaries/cli/build.rs

@@ -1,4 +1,5 @@
 fn main() {
+    pyo3_build_config::add_extension_module_link_args();
     println!(
         "cargo:rustc-env=TARGET={}",
         std::env::var("TARGET").unwrap()

binaries/cli/pyproject.toml

@@ -15,6 +15,14 @@ features = ["python", "pyo3/extension-module"]
 
 [tool.ruff.lint]
 extend-select = [
-  "D",   # pydocstyle
-  "UP"
+  "D",  # pydocstyle
+  "UP",
 ]
+
+[tool.maturin.target.x86_64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"
+
+[tool.maturin.target.aarch64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"

binaries/cli/src/build.rs

@@ -1,116 +0,0 @@
-use dora_core::{
-    config::OperatorId,
-    descriptor::{Descriptor, DescriptorExt, NodeExt, SINGLE_OPERATOR_DEFAULT_ID},
-};
-use dora_message::descriptor::EnvValue;
-use eyre::{eyre, Context};
-use std::{collections::BTreeMap, path::Path, process::Command};
-
-use crate::resolve_dataflow;
-
-pub fn build(dataflow: String, uv: bool) -> eyre::Result<()> {
-    let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-    let descriptor = Descriptor::blocking_read(&dataflow)?;
-    let dataflow_absolute = if dataflow.is_relative() {
-        std::env::current_dir().unwrap().join(dataflow)
-    } else {
-        dataflow.to_owned()
-    };
-    let working_dir = dataflow_absolute.parent().unwrap();
-
-    let default_op_id = OperatorId::from(SINGLE_OPERATOR_DEFAULT_ID.to_string());
-
-    for node in descriptor.nodes {
-        match node.kind()? {
-            dora_core::descriptor::NodeKind::Standard(_) => {
-                run_build_command(node.build.as_deref(), working_dir, uv, node.env.clone())
-                    .with_context(|| {
-                        format!("build command failed for standard node `{}`", node.id)
-                    })?
-            }
-            dora_core::descriptor::NodeKind::Runtime(runtime_node) => {
-                for operator in &runtime_node.operators {
-                    run_build_command(
-                        operator.config.build.as_deref(),
-                        working_dir,
-                        uv,
-                        node.env.clone(),
-                    )
-                    .with_context(|| {
-                        format!(
-                            "build command failed for operator `{}/{}`",
-                            node.id, operator.id
-                        )
-                    })?;
-                }
-            }
-            dora_core::descriptor::NodeKind::Custom(custom_node) => run_build_command(
-                custom_node.build.as_deref(),
-                working_dir,
-                uv,
-                node.env.clone(),
-            )
-            .with_context(|| format!("build command failed for custom node `{}`", node.id))?,
-            dora_core::descriptor::NodeKind::Operator(operator) => run_build_command(
-                operator.config.build.as_deref(),
-                working_dir,
-                uv,
-                node.env.clone(),
-            )
-            .with_context(|| {
-                format!(
-                    "build command failed for operator `{}/{}`",
-                    node.id,
-                    operator.id.as_ref().unwrap_or(&default_op_id)
-                )
-            })?,
-        }
-    }
-
-    Ok(())
-}
-
-fn run_build_command(
-    build: Option<&str>,
-    working_dir: &Path,
-    uv: bool,
-    envs: Option<BTreeMap<String, EnvValue>>,
-) -> eyre::Result<()> {
-    if let Some(build) = build {
-        let lines = build.lines().collect::<Vec<_>>();
-        for build_line in lines {
-            let mut split = build_line.split_whitespace();
-
-            let program = split
-                .next()
-                .ok_or_else(|| eyre!("build command is empty"))?;
-            let mut cmd = if uv && (program == "pip" || program == "pip3") {
-                let mut cmd = Command::new("uv");
-                cmd.arg("pip");
-                cmd
-            } else {
-                Command::new(program)
-            };
-            cmd.args(split);
-
-            // Inject Environment Variables
-            if let Some(envs) = envs.clone() {
-                for (key, value) in envs {
-                    let value = value.to_string();
-                    cmd.env(key, value);
-                }
-            }
-
-            cmd.current_dir(working_dir);
-            let exit_status = cmd
-                .status()
-                .wrap_err_with(|| format!("failed to run `{}`", build))?;
-            if !exit_status.success() {
-                return Err(eyre!("build command `{build_line}` returned {exit_status}"));
-            }
-        }
-        Ok(())
-    } else {
-        Ok(())
-    }
-}

binaries/cli/src/command/build/distributed.rs

@@ -0,0 +1,107 @@
+use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
+use dora_core::descriptor::Descriptor;
+use dora_message::{
+    cli_to_coordinator::ControlRequest,
+    common::{GitSource, LogMessage},
+    coordinator_to_cli::ControlRequestReply,
+    id::NodeId,
+    BuildId,
+};
+use eyre::{bail, Context};
+use std::{
+    collections::BTreeMap,
+    net::{SocketAddr, TcpStream},
+};
+
+use crate::{output::print_log_message, session::DataflowSession};
+
+pub fn build_distributed_dataflow(
+    session: &mut TcpRequestReplyConnection,
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    local_working_dir: Option<std::path::PathBuf>,
+    uv: bool,
+) -> eyre::Result<BuildId> {
+    let build_id = {
+        let reply_raw = session
+            .request(
+                &serde_json::to_vec(&ControlRequest::Build {
+                    session_id: dataflow_session.session_id,
+                    dataflow,
+                    git_sources: git_sources.clone(),
+                    prev_git_sources: dataflow_session.git_sources.clone(),
+                    local_working_dir,
+                    uv,
+                })
+                .unwrap(),
+            )
+            .wrap_err("failed to send start dataflow message")?;
+
+        let result: ControlRequestReply =
+            serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+        match result {
+            ControlRequestReply::DataflowBuildTriggered { build_id } => {
+                eprintln!("dataflow build triggered: {build_id}");
+                build_id
+            }
+            ControlRequestReply::Error(err) => bail!("{err}"),
+            other => bail!("unexpected start dataflow reply: {other:?}"),
+        }
+    };
+    Ok(build_id)
+}
+
+pub fn wait_until_dataflow_built(
+    build_id: BuildId,
+    session: &mut TcpRequestReplyConnection,
+    coordinator_socket: SocketAddr,
+    log_level: log::LevelFilter,
+) -> eyre::Result<BuildId> {
+    // subscribe to log messages
+    let mut log_session = TcpConnection {
+        stream: TcpStream::connect(coordinator_socket)
+            .wrap_err("failed to connect to dora coordinator")?,
+    };
+    log_session
+        .send(
+            &serde_json::to_vec(&ControlRequest::BuildLogSubscribe {
+                build_id,
+                level: log_level,
+            })
+            .wrap_err("failed to serialize message")?,
+        )
+        .wrap_err("failed to send build log subscribe request to coordinator")?;
+    std::thread::spawn(move || {
+        while let Ok(raw) = log_session.receive() {
+            let parsed: eyre::Result<LogMessage> =
+                serde_json::from_slice(&raw).context("failed to parse log message");
+            match parsed {
+                Ok(log_message) => {
+                    print_log_message(log_message, false, true);
+                }
+                Err(err) => {
+                    tracing::warn!("failed to parse log message: {err:?}")
+                }
+            }
+        }
+    });
+
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::WaitForBuild { build_id }).unwrap())
+        .wrap_err("failed to send WaitForBuild message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::DataflowBuildFinished { build_id, result } => match result {
+            Ok(()) => {
+                eprintln!("dataflow build finished successfully");
+                Ok(build_id)
+            }
+            Err(err) => bail!("{err}"),
+        },
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+}

binaries/cli/src/command/build/git.rs

@@ -0,0 +1,45 @@
+use dora_message::{common::GitSource, descriptor::GitRepoRev};
+use eyre::Context;
+
+pub fn fetch_commit_hash(repo_url: String, rev: Option<GitRepoRev>) -> eyre::Result<GitSource> {
+    let mut remote = git2::Remote::create_detached(repo_url.as_bytes())
+        .with_context(|| format!("failed to create git remote for {repo_url}"))?;
+    let connection = remote
+        .connect_auth(git2::Direction::Fetch, None, None)
+        .with_context(|| format!("failed to open connection to {repo_url}"))?;
+    let references = connection
+        .list()
+        .with_context(|| format!("failed to list git references of {repo_url}"))?;
+
+    let expected_name = match &rev {
+        Some(GitRepoRev::Branch(branch)) => format!("refs/heads/{branch}"),
+        Some(GitRepoRev::Tag(tag)) => format!("refs/tags/{tag}"),
+        Some(GitRepoRev::Rev(rev)) => rev.clone(),
+        None => "HEAD".into(),
+    };
+
+    let mut commit_hash = None;
+    for head in references {
+        if head.name() == expected_name {
+            commit_hash = Some(head.oid().to_string());
+            break;
+        }
+    }
+
+    if commit_hash.is_none() {
+        if let Some(GitRepoRev::Rev(rev)) = &rev {
+            // rev might be a commit hash instead of a reference
+            if rev.is_ascii() && rev.bytes().all(|b| b.is_ascii_alphanumeric()) {
+                commit_hash = Some(rev.clone());
+            }
+        }
+    }
+
+    match commit_hash {
+        Some(commit_hash) => Ok(GitSource {
+            repo: repo_url,
+            commit_hash,
+        }),
+        None => eyre::bail!("no matching commit for `{rev:?}`"),
+    }
+}

binaries/cli/src/command/build/local.rs

@@ -0,0 +1,121 @@
+use std::{collections::BTreeMap, path::PathBuf};
+
+use colored::Colorize;
+use dora_core::{
+    build::{BuildInfo, BuildLogger, Builder, GitManager, LogLevelOrStdout, PrevGitSource},
+    descriptor::{Descriptor, DescriptorExt},
+};
+use dora_message::{common::GitSource, id::NodeId};
+use eyre::Context;
+
+use crate::session::DataflowSession;
+
+pub fn build_dataflow_locally(
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    working_dir: PathBuf,
+    uv: bool,
+) -> eyre::Result<BuildInfo> {
+    let runtime = tokio::runtime::Runtime::new()?;
+
+    runtime.block_on(build_dataflow(
+        dataflow,
+        git_sources,
+        dataflow_session,
+        working_dir,
+        uv,
+    ))
+}
+
+async fn build_dataflow(
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    base_working_dir: PathBuf,
+    uv: bool,
+) -> eyre::Result<BuildInfo> {
+    let builder = Builder {
+        session_id: dataflow_session.session_id,
+        base_working_dir,
+        uv,
+    };
+    let nodes = dataflow.resolve_aliases_and_set_defaults()?;
+
+    let mut git_manager = GitManager::default();
+    let prev_git_sources = &dataflow_session.git_sources;
+
+    let mut tasks = Vec::new();
+
+    // build nodes
+    for node in nodes.into_values() {
+        let node_id = node.id.clone();
+        let git_source = git_sources.get(&node_id).cloned();
+        let prev_git_source = prev_git_sources.get(&node_id).cloned();
+        let prev_git = prev_git_source.map(|prev_source| PrevGitSource {
+            still_needed_for_this_build: git_sources.values().any(|s| s == &prev_source),
+            git_source: prev_source,
+        });
+
+        let task = builder
+            .clone()
+            .build_node(
+                node,
+                git_source,
+                prev_git,
+                LocalBuildLogger {
+                    node_id: node_id.clone(),
+                },
+                &mut git_manager,
+            )
+            .await
+            .wrap_err_with(|| format!("failed to build node `{node_id}`"))?;
+        tasks.push((node_id, task));
+    }
+
+    let mut info = BuildInfo {
+        node_working_dirs: Default::default(),
+    };
+    for (node_id, task) in tasks {
+        let node = task
+            .await
+            .with_context(|| format!("failed to build node `{node_id}`"))?;
+        info.node_working_dirs
+            .insert(node_id, node.node_working_dir);
+    }
+    Ok(info)
+}
+
+struct LocalBuildLogger {
+    node_id: NodeId,
+}
+
+impl BuildLogger for LocalBuildLogger {
+    type Clone = Self;
+
+    async fn log_message(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String> + Send,
+    ) {
+        let level = match level.into() {
+            LogLevelOrStdout::LogLevel(level) => match level {
+                log::Level::Error => "ERROR ".red(),
+                log::Level::Warn => "WARN  ".yellow(),
+                log::Level::Info => "INFO  ".green(),
+                log::Level::Debug => "DEBUG ".bright_blue(),
+                log::Level::Trace => "TRACE ".dimmed(),
+            },
+            LogLevelOrStdout::Stdout => "stdout".italic().dimmed(),
+        };
+        let node = self.node_id.to_string().bold().bright_black();
+        let message: String = message.into();
+        println!("{node}: {level}   {message}");
+    }
+
+    async fn try_clone(&self) -> eyre::Result<Self::Clone> {
+        Ok(LocalBuildLogger {
+            node_id: self.node_id.clone(),
+        })
+    }
+}

binaries/cli/src/command/build/mod.rs

@@ -0,0 +1,168 @@
+use communication_layer_request_reply::TcpRequestReplyConnection;
+use dora_core::{
+    descriptor::{CoreNodeKind, CustomNode, Descriptor, DescriptorExt},
+    topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST},
+};
+use dora_message::{descriptor::NodeSource, BuildId};
+use eyre::Context;
+use std::collections::BTreeMap;
+
+use crate::{connect_to_coordinator, resolve_dataflow, session::DataflowSession};
+
+use distributed::{build_distributed_dataflow, wait_until_dataflow_built};
+use local::build_dataflow_locally;
+
+mod distributed;
+mod git;
+mod local;
+
+pub fn build(
+    dataflow: String,
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+    uv: bool,
+    force_local: bool,
+) -> eyre::Result<()> {
+    let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_descriptor =
+        Descriptor::blocking_read(&dataflow_path).wrap_err("Failed to read yaml dataflow")?;
+    let mut dataflow_session =
+        DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
+
+    let mut git_sources = BTreeMap::new();
+    let resolved_nodes = dataflow_descriptor
+        .resolve_aliases_and_set_defaults()
+        .context("failed to resolve nodes")?;
+    for (node_id, node) in resolved_nodes {
+        if let CoreNodeKind::Custom(CustomNode {
+            source: NodeSource::GitBranch { repo, rev },
+            ..
+        }) = node.kind
+        {
+            let source = git::fetch_commit_hash(repo, rev)
+                .with_context(|| format!("failed to find commit hash for `{node_id}`"))?;
+            git_sources.insert(node_id, source);
+        }
+    }
+
+    let session = || connect_to_coordinator_with_defaults(coordinator_addr, coordinator_port);
+
+    let build_kind = if force_local {
+        log::info!("Building locally, as requested through `--force-local`");
+        BuildKind::Local
+    } else if dataflow_descriptor.nodes.iter().all(|n| n.deploy.is_none()) {
+        log::info!("Building locally because dataflow does not contain any `deploy` sections");
+        BuildKind::Local
+    } else if coordinator_addr.is_some() || coordinator_port.is_some() {
+        log::info!("Building through coordinator, using the given coordinator socket information");
+        // explicit coordinator address or port set -> there should be a coordinator running
+        BuildKind::ThroughCoordinator {
+            coordinator_session: session().context("failed to connect to coordinator")?,
+        }
+    } else {
+        match session() {
+            Ok(coordinator_session) => {
+                // we found a local coordinator instance at default port -> use it for building
+                log::info!("Found local dora coordinator instance -> building through coordinator");
+                BuildKind::ThroughCoordinator {
+                    coordinator_session,
+                }
+            }
+            Err(_) => {
+                log::warn!("No dora coordinator instance found -> trying a local build");
+                // no coordinator instance found -> do a local build
+                BuildKind::Local
+            }
+        }
+    };
+
+    match build_kind {
+        BuildKind::Local => {
+            log::info!("running local build");
+            // use dataflow dir as base working dir
+            let local_working_dir = dunce::canonicalize(&dataflow_path)
+                .context("failed to canonicalize dataflow path")?
+                .parent()
+                .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
+                .to_owned();
+            let build_info = build_dataflow_locally(
+                dataflow_descriptor,
+                &git_sources,
+                &dataflow_session,
+                local_working_dir,
+                uv,
+            )?;
+
+            dataflow_session.git_sources = git_sources;
+            // generate a random BuildId and store the associated build info
+            dataflow_session.build_id = Some(BuildId::generate());
+            dataflow_session.local_build = Some(build_info);
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+        }
+        BuildKind::ThroughCoordinator {
+            mut coordinator_session,
+        } => {
+            let local_working_dir = super::local_working_dir(
+                &dataflow_path,
+                &dataflow_descriptor,
+                &mut *coordinator_session,
+            )?;
+            let build_id = build_distributed_dataflow(
+                &mut *coordinator_session,
+                dataflow_descriptor,
+                &git_sources,
+                &dataflow_session,
+                local_working_dir,
+                uv,
+            )?;
+
+            dataflow_session.git_sources = git_sources;
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+
+            // wait until dataflow build is finished
+
+            wait_until_dataflow_built(
+                build_id,
+                &mut *coordinator_session,
+                coordinator_socket(coordinator_addr, coordinator_port),
+                log::LevelFilter::Info,
+            )?;
+
+            dataflow_session.build_id = Some(build_id);
+            dataflow_session.local_build = None;
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+        }
+    };
+
+    Ok(())
+}
+
+enum BuildKind {
+    Local,
+    ThroughCoordinator {
+        coordinator_session: Box<TcpRequestReplyConnection>,
+    },
+}
+
+fn connect_to_coordinator_with_defaults(
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+) -> std::io::Result<Box<TcpRequestReplyConnection>> {
+    let coordinator_socket = coordinator_socket(coordinator_addr, coordinator_port);
+    connect_to_coordinator(coordinator_socket)
+}
+
+fn coordinator_socket(
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+) -> std::net::SocketAddr {
+    let coordinator_addr = coordinator_addr.unwrap_or(LOCALHOST);
+    let coordinator_port = coordinator_port.unwrap_or(DORA_COORDINATOR_PORT_CONTROL_DEFAULT);
+    (coordinator_addr, coordinator_port).into()
+}

binaries/cli/src/command/mod.rs

@@ -0,0 +1,60 @@
+pub use build::build;
+pub use logs::logs;
+pub use run::run;
+pub use start::start;
+
+use std::path::{Path, PathBuf};
+
+use communication_layer_request_reply::TcpRequestReplyConnection;
+use dora_core::descriptor::Descriptor;
+use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
+use eyre::{bail, Context, ContextCompat};
+
+mod build;
+pub mod check;
+mod logs;
+mod run;
+mod start;
+pub mod up;
+
+fn local_working_dir(
+    dataflow_path: &Path,
+    dataflow_descriptor: &Descriptor,
+    coordinator_session: &mut TcpRequestReplyConnection,
+) -> eyre::Result<Option<PathBuf>> {
+    Ok(
+        if dataflow_descriptor
+            .nodes
+            .iter()
+            .all(|n| n.deploy.as_ref().map(|d| d.machine.as_ref()).is_none())
+            && cli_and_daemon_on_same_machine(coordinator_session)?
+        {
+            Some(
+                dunce::canonicalize(dataflow_path)
+                    .context("failed to canonicalize dataflow file path")?
+                    .parent()
+                    .context("dataflow path has no parent dir")?
+                    .to_owned(),
+            )
+        } else {
+            None
+        },
+    )
+}
+
+fn cli_and_daemon_on_same_machine(session: &mut TcpRequestReplyConnection) -> eyre::Result<bool> {
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::CliAndDefaultDaemonOnSameMachine).unwrap())
+        .wrap_err("failed to send start dataflow message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::CliAndDefaultDaemonIps {
+            default_daemon,
+            cli,
+        } => Ok(default_daemon.is_some() && default_daemon == cli),
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+}

binaries/cli/src/command/run.rs

@@ -0,0 +1,34 @@
+use dora_daemon::{flume, Daemon, LogDestination};
+use eyre::Context;
+use tokio::runtime::Builder;
+
+use crate::{
+    handle_dataflow_result, output::print_log_message, resolve_dataflow, session::DataflowSession,
+};
+
+pub fn run(dataflow: String, uv: bool) -> Result<(), eyre::Error> {
+    let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_session =
+        DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
+    let rt = Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .context("tokio runtime failed")?;
+
+    let (log_tx, log_rx) = flume::bounded(100);
+    std::thread::spawn(move || {
+        for message in log_rx {
+            print_log_message(message, false, false);
+        }
+    });
+
+    let result = rt.block_on(Daemon::run_dataflow(
+        &dataflow_path,
+        dataflow_session.build_id,
+        dataflow_session.local_build,
+        dataflow_session.session_id,
+        uv,
+        LogDestination::Channel { sender: log_tx },
+    ))?;
+    handle_dataflow_result(result, None)
+}

binaries/cli/src/attach.rs → binaries/cli/src/command/start/attach.rs

@@ -1,4 +1,3 @@
-use colored::Colorize;
 use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
 use dora_core::descriptor::{resolve_path, CoreNodeKind, Descriptor, DescriptorExt};
 use dora_message::cli_to_coordinator::ControlRequest;
@@ -16,6 +15,7 @@ use tracing::{error, info};
 use uuid::Uuid;
 
 use crate::handle_dataflow_result;
+use crate::output::print_log_message;
 
 pub fn attach_dataflow(
     dataflow: Descriptor,
@@ -33,6 +33,8 @@ pub fn attach_dataflow(
 
     let nodes = dataflow.resolve_aliases_and_set_defaults()?;
 
+    let print_daemon_name = nodes.values().any(|n| n.deploy.is_some());
+
     let working_dir = dataflow_path
         .canonicalize()
         .context("failed to canonicalize dataflow path")?
@@ -155,39 +157,7 @@ pub fn attach_dataflow(
             },
             Ok(AttachEvent::Control(control_request)) => control_request,
             Ok(AttachEvent::Log(Ok(log_message))) => {
-                let LogMessage {
-                    dataflow_id,
-                    node_id,
-                    daemon_id,
-                    level,
-                    target,
-                    module_path: _,
-                    file: _,
-                    line: _,
-                    message,
-                } = log_message;
-                let level = match level {
-                    log::Level::Error => "ERROR".red(),
-                    log::Level::Warn => "WARN ".yellow(),
-                    log::Level::Info => "INFO ".green(),
-                    other => format!("{other:5}").normal(),
-                };
-                let dataflow = format!(" dataflow `{dataflow_id}`").cyan();
-                let daemon = match daemon_id {
-                    Some(id) => format!(" on daemon `{id}`"),
-                    None => " on default daemon".to_string(),
-                }
-                .bright_black();
-                let node = match node_id {
-                    Some(node_id) => format!(" {node_id}").bold(),
-                    None => "".normal(),
-                };
-                let target = match target {
-                    Some(target) => format!(" {target}").dimmed(),
-                    None => "".normal(),
-                };
-
-                println!("{level}{dataflow}{daemon}{node}{target}: {message}");
+                print_log_message(log_message, false, print_daemon_name);
                 continue;
             }
             Ok(AttachEvent::Log(Err(err))) => {
@@ -202,7 +172,7 @@ pub fn attach_dataflow(
         let result: ControlRequestReply =
             serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
         match result {
-            ControlRequestReply::DataflowStarted { uuid: _ } => (),
+            ControlRequestReply::DataflowSpawned { uuid: _ } => (),
             ControlRequestReply::DataflowStopped { uuid, result } => {
                 info!("dataflow {uuid} stopped");
                 break handle_dataflow_result(result, Some(uuid));

binaries/cli/src/command/start/mod.rs

@@ -0,0 +1,170 @@
+use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
+use dora_core::descriptor::{Descriptor, DescriptorExt};
+use dora_message::{
+    cli_to_coordinator::ControlRequest, common::LogMessage, coordinator_to_cli::ControlRequestReply,
+};
+use eyre::{bail, Context};
+use std::{
+    net::{SocketAddr, TcpStream},
+    path::PathBuf,
+};
+use uuid::Uuid;
+
+use crate::{
+    connect_to_coordinator, output::print_log_message, resolve_dataflow, session::DataflowSession,
+};
+use attach::attach_dataflow;
+
+mod attach;
+
+pub fn start(
+    dataflow: String,
+    name: Option<String>,
+    coordinator_socket: SocketAddr,
+    attach: bool,
+    detach: bool,
+    hot_reload: bool,
+    uv: bool,
+) -> eyre::Result<()> {
+    let (dataflow, dataflow_descriptor, mut session, dataflow_id) =
+        start_dataflow(dataflow, name, coordinator_socket, uv)?;
+
+    let attach = match (attach, detach) {
+        (true, true) => eyre::bail!("both `--attach` and `--detach` are given"),
+        (true, false) => true,
+        (false, true) => false,
+        (false, false) => {
+            println!("attaching to dataflow (use `--detach` to run in background)");
+            true
+        }
+    };
+
+    if attach {
+        let log_level = env_logger::Builder::new()
+            .filter_level(log::LevelFilter::Info)
+            .parse_default_env()
+            .build()
+            .filter();
+
+        attach_dataflow(
+            dataflow_descriptor,
+            dataflow,
+            dataflow_id,
+            &mut *session,
+            hot_reload,
+            coordinator_socket,
+            log_level,
+        )
+    } else {
+        let print_daemon_name = dataflow_descriptor.nodes.iter().any(|n| n.deploy.is_some());
+        // wait until dataflow is started
+        wait_until_dataflow_started(
+            dataflow_id,
+            &mut session,
+            coordinator_socket,
+            log::LevelFilter::Info,
+            print_daemon_name,
+        )
+    }
+}
+
+fn start_dataflow(
+    dataflow: String,
+    name: Option<String>,
+    coordinator_socket: SocketAddr,
+    uv: bool,
+) -> Result<(PathBuf, Descriptor, Box<TcpRequestReplyConnection>, Uuid), eyre::Error> {
+    let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_descriptor =
+        Descriptor::blocking_read(&dataflow).wrap_err("Failed to read yaml dataflow")?;
+    let dataflow_session =
+        DataflowSession::read_session(&dataflow).context("failed to read DataflowSession")?;
+
+    let mut session = connect_to_coordinator(coordinator_socket)
+        .wrap_err("failed to connect to dora coordinator")?;
+
+    let local_working_dir =
+        super::local_working_dir(&dataflow, &dataflow_descriptor, &mut *session)?;
+
+    let dataflow_id = {
+        let dataflow = dataflow_descriptor.clone();
+        let session: &mut TcpRequestReplyConnection = &mut *session;
+        let reply_raw = session
+            .request(
+                &serde_json::to_vec(&ControlRequest::Start {
+                    build_id: dataflow_session.build_id,
+                    session_id: dataflow_session.session_id,
+                    dataflow,
+                    name,
+                    local_working_dir,
+                    uv,
+                })
+                .unwrap(),
+            )
+            .wrap_err("failed to send start dataflow message")?;
+
+        let result: ControlRequestReply =
+            serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+        match result {
+            ControlRequestReply::DataflowStartTriggered { uuid } => {
+                eprintln!("dataflow start triggered: {uuid}");
+                uuid
+            }
+            ControlRequestReply::Error(err) => bail!("{err}"),
+            other => bail!("unexpected start dataflow reply: {other:?}"),
+        }
+    };
+    Ok((dataflow, dataflow_descriptor, session, dataflow_id))
+}
+
+fn wait_until_dataflow_started(
+    dataflow_id: Uuid,
+    session: &mut Box<TcpRequestReplyConnection>,
+    coordinator_addr: SocketAddr,
+    log_level: log::LevelFilter,
+    print_daemon_id: bool,
+) -> eyre::Result<()> {
+    // subscribe to log messages
+    let mut log_session = TcpConnection {
+        stream: TcpStream::connect(coordinator_addr)
+            .wrap_err("failed to connect to dora coordinator")?,
+    };
+    log_session
+        .send(
+            &serde_json::to_vec(&ControlRequest::LogSubscribe {
+                dataflow_id,
+                level: log_level,
+            })
+            .wrap_err("failed to serialize message")?,
+        )
+        .wrap_err("failed to send log subscribe request to coordinator")?;
+    std::thread::spawn(move || {
+        while let Ok(raw) = log_session.receive() {
+            let parsed: eyre::Result<LogMessage> =
+                serde_json::from_slice(&raw).context("failed to parse log message");
+            match parsed {
+                Ok(log_message) => {
+                    print_log_message(log_message, false, print_daemon_id);
+                }
+                Err(err) => {
+                    tracing::warn!("failed to parse log message: {err:?}")
+                }
+            }
+        }
+    });
+
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::WaitForSpawn { dataflow_id }).unwrap())
+        .wrap_err("failed to send start dataflow message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::DataflowSpawned { uuid } => {
+            eprintln!("dataflow started: {uuid}");
+        }
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+    Ok(())
+}

binaries/cli/src/up.rs → binaries/cli/src/command/up.rs

@@ -1,4 +1,4 @@
-use crate::{check::daemon_running, connect_to_coordinator, LOCALHOST};
+use crate::{command::check::daemon_running, connect_to_coordinator, LOCALHOST};
 use dora_core::topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT;
 use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
 use eyre::{bail, Context, ContextCompat};

binaries/cli/src/lib.rs

@@ -1,4 +1,3 @@
-use attach::attach_dataflow;
 use colored::Colorize;
 use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection};
 use dora_coordinator::Event;
@@ -9,7 +8,7 @@ use dora_core::{
         DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT,
     },
 };
-use dora_daemon::Daemon;
+use dora_daemon::{Daemon, LogDestination};
 use dora_download::download_file;
 use dora_message::{
     cli_to_coordinator::ControlRequest,
@@ -31,14 +30,12 @@ use tokio::runtime::Builder;
 use tracing::level_filters::LevelFilter;
 use uuid::Uuid;
 
-mod attach;
-mod build;
-mod check;
+pub mod command;
 mod formatting;
 mod graph;
-mod logs;
+pub mod output;
+pub mod session;
 mod template;
-mod up;
 
 const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
 const LISTEN_WILDCARD: IpAddr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
@@ -82,9 +79,18 @@ enum Command {
         /// Path to the dataflow descriptor file
         #[clap(value_name = "PATH")]
         dataflow: String,
+        /// Address of the dora coordinator
+        #[clap(long, value_name = "IP")]
+        coordinator_addr: Option<IpAddr>,
+        /// Port number of the coordinator control server
+        #[clap(long, value_name = "PORT")]
+        coordinator_port: Option<u16>,
         // Use UV to build nodes.
         #[clap(long, action)]
         uv: bool,
+        // Run build on local machine
+        #[clap(long, action)]
+        local: bool,
     },
     /// Generate a new project or node. Choose the language between Rust, Python, C or C++.
     New {
@@ -292,14 +298,16 @@ enum Lang {
 }
 
 pub fn lib_main(args: Args) {
-    if let Err(err) = run(args) {
+    if let Err(err) = run_cli(args) {
         eprintln!("\n\n{}", "[ERROR]".bold().red());
         eprintln!("{err:?}");
         std::process::exit(1);
     }
 }
 
-fn run(args: Args) -> eyre::Result<()> {
+fn run_cli(args: Args) -> eyre::Result<()> {
+    tracing_log::LogTracer::init()?;
+
     #[cfg(feature = "tracing")]
     match &args.command {
         Command::Daemon {
@@ -334,7 +342,7 @@ fn run(args: Args) -> eyre::Result<()> {
                 .build()
                 .wrap_err("failed to set up tracing subscriber")?;
         }
-        Command::Run { .. } => {
+        Command::Run { .. } | Command::Build { .. } => {
             let log_level = std::env::var("RUST_LOG").ok().unwrap_or("info".to_string());
             TracingBuilder::new("run")
                 .with_stdout(log_level)
@@ -349,12 +357,6 @@ fn run(args: Args) -> eyre::Result<()> {
         }
     };
 
-    let log_level = env_logger::Builder::new()
-        .filter_level(log::LevelFilter::Info)
-        .parse_default_env()
-        .build()
-        .filter();
-
     match args.command {
         Command::Check {
             dataflow,
@@ -369,9 +371,9 @@ fn run(args: Args) -> eyre::Result<()> {
                     .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
                     .to_owned();
                 Descriptor::blocking_read(&dataflow)?.check(&working_dir)?;
-                check::check_environment((coordinator_addr, coordinator_port).into())?
+                command::check::check_environment((coordinator_addr, coordinator_port).into())?
             }
-            None => check::check_environment((coordinator_addr, coordinator_port).into())?,
+            None => command::check::check_environment((coordinator_addr, coordinator_port).into())?,
         },
         Command::Graph {
             dataflow,
@@ -380,24 +382,20 @@ fn run(args: Args) -> eyre::Result<()> {
         } => {
             graph::create(dataflow, mermaid, open)?;
         }
-        Command::Build { dataflow, uv } => {
-            build::build(dataflow, uv)?;
-        }
+        Command::Build {
+            dataflow,
+            coordinator_addr,
+            coordinator_port,
+            uv,
+            local,
+        } => command::build(dataflow, coordinator_addr, coordinator_port, uv, local)?,
         Command::New {
             args,
             internal_create_with_path_dependencies,
         } => template::create(args, internal_create_with_path_dependencies)?,
-        Command::Run { dataflow, uv } => {
-            let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-            let rt = Builder::new_multi_thread()
-                .enable_all()
-                .build()
-                .context("tokio runtime failed")?;
-            let result = rt.block_on(Daemon::run_dataflow(&dataflow_path, uv))?;
-            handle_dataflow_result(result, None)?
-        }
+        Command::Run { dataflow, uv } => command::run(dataflow, uv)?,
         Command::Up { config } => {
-            up::up(config.as_deref())?;
+            command::up::up(config.as_deref())?;
         }
         Command::Logs {
             dataflow,
@@ -412,15 +410,16 @@ fn run(args: Args) -> eyre::Result<()> {
             if let Some(dataflow) = dataflow {
                 let uuid = Uuid::parse_str(&dataflow).ok();
                 let name = if uuid.is_some() { None } else { Some(dataflow) };
-                logs::logs(&mut *session, uuid, name, node)?
+                command::logs(&mut *session, uuid, name, node)?
             } else {
-                let active = list.get_active();
+                let active: Vec<dora_message::coordinator_to_cli::DataflowIdAndName> =
+                    list.get_active();
                 let uuid = match &active[..] {
                     [] => bail!("No dataflows are running"),
                     [uuid] => uuid.clone(),
                     _ => inquire::Select::new("Choose dataflow to show logs:", active).prompt()?,
                 };
-                logs::logs(&mut *session, Some(uuid.uuid), None, node)?
+                command::logs(&mut *session, Some(uuid.uuid), None, node)?
             }
         }
         Command::Start {
@@ -433,48 +432,16 @@ fn run(args: Args) -> eyre::Result<()> {
             hot_reload,
             uv,
         } => {
-            let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-            let dataflow_descriptor =
-                Descriptor::blocking_read(&dataflow).wrap_err("Failed to read yaml dataflow")?;
-            let working_dir = dataflow
-                .canonicalize()
-                .context("failed to canonicalize dataflow path")?
-                .parent()
-                .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
-                .to_owned();
-
             let coordinator_socket = (coordinator_addr, coordinator_port).into();
-            let mut session = connect_to_coordinator(coordinator_socket)
-                .wrap_err("failed to connect to dora coordinator")?;
-            let dataflow_id = start_dataflow(
-                dataflow_descriptor.clone(),
+            command::start(
+                dataflow,
                 name,
-                working_dir,
-                &mut *session,
+                coordinator_socket,
+                attach,
+                detach,
+                hot_reload,
                 uv,
-            )?;
-
-            let attach = match (attach, detach) {
-                (true, true) => eyre::bail!("both `--attach` and `--detach` are given"),
-                (true, false) => true,
-                (false, true) => false,
-                (false, false) => {
-                    println!("attaching to dataflow (use `--detach` to run in background)");
-                    true
-                }
-            };
-
-            if attach {
-                attach_dataflow(
-                    dataflow_descriptor,
-                    dataflow,
-                    dataflow_id,
-                    &mut *session,
-                    hot_reload,
-                    coordinator_socket,
-                    log_level,
-                )?
-            }
+            )?
         }
         Command::List {
             coordinator_addr,
@@ -504,7 +471,7 @@ fn run(args: Args) -> eyre::Result<()> {
             config,
             coordinator_addr,
             coordinator_port,
-        } => up::destroy(
+        } => command::up::destroy(
             config.as_deref(),
             (coordinator_addr, coordinator_port).into(),
         )?,
@@ -554,8 +521,13 @@ fn run(args: Args) -> eyre::Result<()> {
                                 coordinator_addr
                             );
                         }
+                        let dataflow_session =
+                            DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
 
-                        let result = Daemon::run_dataflow(&dataflow_path, false).await?;
+                        let result = Daemon::run_dataflow(&dataflow_path,
+                            dataflow_session.build_id, dataflow_session.local_build, dataflow_session.session_id, false,
+                            LogDestination::Tracing,
+                        ).await?;
                         handle_dataflow_result(result, None)
                     }
                     None => {
@@ -682,37 +654,6 @@ fn run(args: Args) -> eyre::Result<()> {
     Ok(())
 }
 
-fn start_dataflow(
-    dataflow: Descriptor,
-    name: Option<String>,
-    local_working_dir: PathBuf,
-    session: &mut TcpRequestReplyConnection,
-    uv: bool,
-) -> Result<Uuid, eyre::ErrReport> {
-    let reply_raw = session
-        .request(
-            &serde_json::to_vec(&ControlRequest::Start {
-                dataflow,
-                name,
-                local_working_dir,
-                uv,
-            })
-            .unwrap(),
-        )
-        .wrap_err("failed to send start dataflow message")?;
-
-    let result: ControlRequestReply =
-        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
-    match result {
-        ControlRequestReply::DataflowStarted { uuid } => {
-            eprintln!("{uuid}");
-            Ok(uuid)
-        }
-        ControlRequestReply::Error(err) => bail!("{err}"),
-        other => bail!("unexpected start dataflow reply: {other:?}"),
-    }
-}
-
 fn stop_dataflow_interactive(
     grace_duration: Option<Duration>,
     session: &mut TcpRequestReplyConnection,
@@ -863,6 +804,8 @@ use pyo3::{
     wrap_pyfunction, Bound, PyResult, Python,
 };
 
+use crate::session::DataflowSession;
+
 #[cfg(feature = "python")]
 #[pyfunction]
 fn py_main(_py: Python) -> PyResult<()> {

binaries/cli/src/output.rs

@@ -0,0 +1,62 @@
+use colored::Colorize;
+use dora_core::build::LogLevelOrStdout;
+use dora_message::common::LogMessage;
+
+pub fn print_log_message(
+    log_message: LogMessage,
+    print_dataflow_id: bool,
+    print_daemon_name: bool,
+) {
+    let LogMessage {
+        build_id: _,
+        dataflow_id,
+        node_id,
+        daemon_id,
+        level,
+        target,
+        module_path: _,
+        file: _,
+        line: _,
+        message,
+    } = log_message;
+    let level = match level {
+        LogLevelOrStdout::LogLevel(level) => match level {
+            log::Level::Error => "ERROR ".red(),
+            log::Level::Warn => "WARN  ".yellow(),
+            log::Level::Info => "INFO  ".green(),
+            log::Level::Debug => "DEBUG ".bright_blue(),
+            log::Level::Trace => "TRACE ".dimmed(),
+        },
+        LogLevelOrStdout::Stdout => "stdout".bright_blue().italic().dimmed(),
+    };
+
+    let dataflow = match dataflow_id {
+        Some(dataflow_id) if print_dataflow_id => format!("dataflow `{dataflow_id}` ").cyan(),
+        _ => String::new().cyan(),
+    };
+    let daemon = match daemon_id {
+        Some(id) if print_daemon_name => match id.machine_id() {
+            Some(machine_id) => format!("on daemon `{machine_id}`"),
+            None => "on default daemon ".to_string(),
+        },
+        None if print_daemon_name => "on default daemon".to_string(),
+        _ => String::new(),
+    }
+    .bright_black();
+    let colon = ":".bright_black().bold();
+    let node = match node_id {
+        Some(node_id) => {
+            let node_id = node_id.to_string().dimmed().bold();
+            let padding = if daemon.is_empty() { "" } else { " " };
+            format!("{node_id}{padding}{daemon}{colon} ")
+        }
+        None if daemon.is_empty() => "".into(),
+        None => format!("{daemon}{colon} "),
+    };
+    let target = match target {
+        Some(target) => format!("{target} ").dimmed(),
+        None => "".normal(),
+    };
+
+    println!("{node}{level} {target}{dataflow}   {message}");
+}

binaries/cli/src/session.rs

@@ -0,0 +1,98 @@
+use std::{
+    collections::BTreeMap,
+    path::{Path, PathBuf},
+};
+
+use dora_core::build::BuildInfo;
+use dora_message::{common::GitSource, id::NodeId, BuildId, SessionId};
+use eyre::{Context, ContextCompat};
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct DataflowSession {
+    pub build_id: Option<BuildId>,
+    pub session_id: SessionId,
+    pub git_sources: BTreeMap<NodeId, GitSource>,
+    pub local_build: Option<BuildInfo>,
+}
+
+impl Default for DataflowSession {
+    fn default() -> Self {
+        Self {
+            build_id: None,
+            session_id: SessionId::generate(),
+            git_sources: Default::default(),
+            local_build: Default::default(),
+        }
+    }
+}
+
+impl DataflowSession {
+    pub fn read_session(dataflow_path: &Path) -> eyre::Result<Self> {
+        let session_file = session_file_path(dataflow_path)?;
+        if session_file.exists() {
+            if let Ok(parsed) = deserialize(&session_file) {
+                return Ok(parsed);
+            } else {
+                tracing::warn!("failed to read dataflow session file, regenerating (you might need to run `dora build` again)");
+            }
+        }
+
+        let default_session = DataflowSession::default();
+        default_session.write_out_for_dataflow(dataflow_path)?;
+        Ok(default_session)
+    }
+
+    pub fn write_out_for_dataflow(&self, dataflow_path: &Path) -> eyre::Result<()> {
+        let session_file = session_file_path(dataflow_path)?;
+        let filename = session_file
+            .file_name()
+            .context("session file has no file name")?
+            .to_str()
+            .context("session file name is no utf8")?;
+        if let Some(parent) = session_file.parent() {
+            std::fs::create_dir_all(parent).context("failed to create out dir")?;
+        }
+        std::fs::write(&session_file, self.serialize()?)
+            .context("failed to write dataflow session file")?;
+        let gitignore = session_file.with_file_name(".gitignore");
+        if gitignore.exists() {
+            let existing =
+                std::fs::read_to_string(&gitignore).context("failed to read gitignore")?;
+            if !existing
+                .lines()
+                .any(|l| l.split_once('/') == Some(("", filename)))
+            {
+                let new = existing + &format!("\n/{filename}\n");
+                std::fs::write(gitignore, new).context("failed to update gitignore")?;
+            }
+        } else {
+            std::fs::write(gitignore, format!("/{filename}\n"))
+                .context("failed to write gitignore")?;
+        }
+        Ok(())
+    }
+
+    fn serialize(&self) -> eyre::Result<String> {
+        serde_yaml::to_string(&self).context("failed to serialize dataflow session file")
+    }
+}
+
+fn deserialize(session_file: &Path) -> eyre::Result<DataflowSession> {
+    std::fs::read_to_string(session_file)
+        .context("failed to read DataflowSession file")
+        .and_then(|s| {
+            serde_yaml::from_str(&s).context("failed to deserialize DataflowSession file")
+        })
+}
+
+fn session_file_path(dataflow_path: &Path) -> eyre::Result<PathBuf> {
+    let file_stem = dataflow_path
+        .file_stem()
+        .wrap_err("dataflow path has no file stem")?
+        .to_str()
+        .wrap_err("dataflow file stem is not valid utf-8")?;
+    let session_file = dataflow_path
+        .with_file_name("out")
+        .join(format!("{file_stem}.dora-session.yaml"));
+    Ok(session_file)
+}

binaries/cli/src/template/c/cmake-template.txt

@@ -64,16 +64,16 @@ link_directories(${dora_link_dirs})
 add_executable(talker_1 talker_1/node.c)
 add_dependencies(talker_1 Dora_c)
 target_include_directories(talker_1 PRIVATE ${dora_c_include_dir})
-target_link_libraries(talker_1 dora_node_api_c m)
+target_link_libraries(talker_1 dora_node_api_c m z)
 
 add_executable(talker_2 talker_2/node.c)
 add_dependencies(talker_2 Dora_c)
 target_include_directories(talker_2 PRIVATE ${dora_c_include_dir})
-target_link_libraries(talker_2 dora_node_api_c m)
+target_link_libraries(talker_2 dora_node_api_c m z)
 
 add_executable(listener_1 listener_1/node.c)
 add_dependencies(listener_1 Dora_c)
 target_include_directories(listener_1 PRIVATE ${dora_c_include_dir})
-target_link_libraries(listener_1 dora_node_api_c m)
+target_link_libraries(listener_1 dora_node_api_c m z)
 
-install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)

binaries/cli/src/template/cxx/cmake-template.txt

@@ -70,16 +70,16 @@ link_directories(${dora_link_dirs})
 add_executable(talker_1 talker_1/node.cc ${node_bridge})
 add_dependencies(talker_1 Dora_cxx)
 target_include_directories(talker_1 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(talker_1 dora_node_api_cxx)
+target_link_libraries(talker_1 dora_node_api_cxx z)
 
 add_executable(talker_2 talker_2/node.cc ${node_bridge})
 add_dependencies(talker_2 Dora_cxx)
 target_include_directories(talker_2 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(talker_2 dora_node_api_cxx)
+target_link_libraries(talker_2 dora_node_api_cxx z)
 
 add_executable(listener_1 listener_1/node.cc ${node_bridge})
 add_dependencies(listener_1 Dora_cxx)
 target_include_directories(listener_1 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(listener_1 dora_node_api_cxx)
+target_link_libraries(listener_1 dora_node_api_cxx z)
 
-install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)

binaries/coordinator/Cargo.toml

@@ -19,7 +19,7 @@ futures = "0.3.21"
 tokio = { version = "1.24.2", features = ["full"] }
 tokio-stream = { version = "0.1.8", features = ["io-util", "net"] }
 uuid = { version = "1.2.1" }
-dora-core = { workspace = true }
+dora-core = { workspace = true, features = ["build"] }
 tracing = "0.1.36"
 dora-tracing = { workspace = true, optional = true }
 futures-concurrency = "7.1.0"

binaries/coordinator/src/control.rs

@@ -2,7 +2,9 @@ use crate::{
     tcp_utils::{tcp_receive, tcp_send},
     Event,
 };
-use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
+use dora_message::{
+    cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply, BuildId,
+};
 use eyre::{eyre, Context};
 use futures::{
     future::{self, Either},
@@ -79,6 +81,7 @@ async fn handle_requests(
     tx: mpsc::Sender<ControlEvent>,
     _finish_tx: mpsc::Sender<()>,
 ) {
+    let peer_addr = connection.peer_addr().ok();
     loop {
         let next_request = tcp_receive(&mut connection).map(Either::Left);
         let coordinator_stopped = tx.closed().map(Either::Right);
@@ -114,11 +117,29 @@ async fn handle_requests(
             break;
         }
 
-        let result = match request {
+        if let Ok(ControlRequest::BuildLogSubscribe { build_id, level }) = request {
+            let _ = tx
+                .send(ControlEvent::BuildLogSubscribe {
+                    build_id,
+                    level,
+                    connection,
+                })
+                .await;
+            break;
+        }
+
+        let mut result = match request {
             Ok(request) => handle_request(request, &tx).await,
             Err(err) => Err(err),
         };
 
+        if let Ok(ControlRequestReply::CliAndDefaultDaemonIps { cli, .. }) = &mut result {
+            if cli.is_none() {
+                // fill cli IP address in reply
+                *cli = peer_addr.map(|s| s.ip());
+            }
+        }
+
         let reply = result.unwrap_or_else(|err| ControlRequestReply::Error(format!("{err:?}")));
         let serialized: Vec<u8> =
             match serde_json::to_vec(&reply).wrap_err("failed to serialize ControlRequestReply") {
@@ -155,7 +176,7 @@ async fn handle_request(
 ) -> eyre::Result<ControlRequestReply> {
     let (reply_tx, reply_rx) = oneshot::channel();
     let event = ControlEvent::IncomingRequest {
-        request,
+        request: request.clone(),
         reply_sender: reply_tx,
     };
 
@@ -165,7 +186,7 @@ async fn handle_request(
 
     reply_rx
         .await
-        .unwrap_or(Ok(ControlRequestReply::CoordinatorStopped))
+        .wrap_err_with(|| format!("no coordinator reply to {request:?}"))?
 }
 
 #[derive(Debug)]
@@ -179,6 +200,11 @@ pub enum ControlEvent {
         level: log::LevelFilter,
         connection: TcpStream,
     },
+    BuildLogSubscribe {
+        build_id: BuildId,
+        level: log::LevelFilter,
+        connection: TcpStream,
+    },
     Error(eyre::Report),
 }
 

binaries/coordinator/src/lib.rs

@@ -5,22 +5,27 @@ use crate::{
 pub use control::ControlEvent;
 use dora_core::{
     config::{NodeId, OperatorId},
+    descriptor::DescriptorExt,
     uhlc::{self, HLC},
 };
 use dora_message::{
     cli_to_coordinator::ControlRequest,
-    common::DaemonId,
+    common::{DaemonId, GitSource},
     coordinator_to_cli::{
         ControlRequestReply, DataflowIdAndName, DataflowList, DataflowListEntry, DataflowResult,
         DataflowStatus, LogLevel, LogMessage,
     },
-    coordinator_to_daemon::{DaemonCoordinatorEvent, RegisterResult, Timestamped},
+    coordinator_to_daemon::{
+        BuildDataflowNodes, DaemonCoordinatorEvent, RegisterResult, Timestamped,
+    },
     daemon_to_coordinator::{DaemonCoordinatorReply, DataflowDaemonResult},
     descriptor::{Descriptor, ResolvedNode},
+    BuildId, DataflowId, SessionId,
 };
 use eyre::{bail, eyre, ContextCompat, Result, WrapErr};
 use futures::{future::join_all, stream::FuturesUnordered, Future, Stream, StreamExt};
 use futures_concurrency::stream::Merge;
+use itertools::Itertools;
 use log_subscriber::LogSubscriber;
 use run::SpawnedDataflow;
 use std::{
@@ -30,7 +35,11 @@ use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
-use tokio::{net::TcpStream, sync::mpsc, task::JoinHandle};
+use tokio::{
+    net::TcpStream,
+    sync::{mpsc, oneshot},
+    task::JoinHandle,
+};
 use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};
 use uuid::Uuid;
 
@@ -135,6 +144,10 @@ impl DaemonConnections {
         }
     }
 
+    fn get(&self, id: &DaemonId) -> Option<&DaemonConnection> {
+        self.daemons.get(id)
+    }
+
     fn get_mut(&mut self, id: &DaemonId) -> Option<&mut DaemonConnection> {
         self.daemons.get_mut(id)
     }
@@ -157,10 +170,6 @@ impl DaemonConnections {
         self.daemons.keys()
     }
 
-    fn iter(&self) -> impl Iterator<Item = (&DaemonId, &DaemonConnection)> {
-        self.daemons.iter()
-    }
-
     fn iter_mut(&mut self) -> impl Iterator<Item = (&DaemonId, &mut DaemonConnection)> {
         self.daemons.iter_mut()
     }
@@ -194,13 +203,20 @@ async fn start_inner(
 
     let mut events = (abortable_events, daemon_events).merge();
 
-    let mut running_dataflows: HashMap<Uuid, RunningDataflow> = HashMap::new();
-    let mut dataflow_results: HashMap<Uuid, BTreeMap<DaemonId, DataflowDaemonResult>> =
+    let mut running_builds: HashMap<BuildId, RunningBuild> = HashMap::new();
+    let mut finished_builds: HashMap<BuildId, CachedResult> = HashMap::new();
+
+    let mut running_dataflows: HashMap<DataflowId, RunningDataflow> = HashMap::new();
+    let mut dataflow_results: HashMap<DataflowId, BTreeMap<DaemonId, DataflowDaemonResult>> =
         HashMap::new();
-    let mut archived_dataflows: HashMap<Uuid, ArchivedDataflow> = HashMap::new();
+    let mut archived_dataflows: HashMap<DataflowId, ArchivedDataflow> = HashMap::new();
     let mut daemon_connections = DaemonConnections::default();
 
     while let Some(event) = events.next().await {
+        // used below for measuring the event handling duration
+        let start = Instant::now();
+        let event_kind = event.kind();
+
         if event.log() {
             tracing::trace!("Handling event {event:?}");
         }
@@ -347,12 +363,13 @@ async fn start_inner(
                                 let mut finished_dataflow = entry.remove();
                                 let dataflow_id = finished_dataflow.uuid;
                                 send_log_message(
-                                    &mut finished_dataflow,
+                                    &mut finished_dataflow.log_subscribers,
                                     &LogMessage {
-                                        dataflow_id,
+                                        build_id: None,
+                                        dataflow_id: Some(dataflow_id),
                                         node_id: None,
                                         daemon_id: None,
-                                        level: LogLevel::Info,
+                                        level: LogLevel::Info.into(),
                                         target: Some("coordinator".into()),
                                         module_path: None,
                                         file: None,
@@ -371,9 +388,15 @@ async fn start_inner(
                                             DataflowResult::ok_empty(uuid, clock.new_timestamp())
                                         }),
                                 };
-                                for sender in finished_dataflow.reply_senders {
+                                for sender in finished_dataflow.stop_reply_senders {
                                     let _ = sender.send(Ok(reply.clone()));
                                 }
+                                if !matches!(
+                                    finished_dataflow.spawn_result,
+                                    CachedResult::Cached { .. }
+                                ) {
+                                    log::error!("pending spawn result on dataflow finish");
+                                }
                             }
                         }
                         std::collections::hash_map::Entry::Vacant(_) => {
@@ -389,7 +412,54 @@ async fn start_inner(
                     reply_sender,
                 } => {
                     match request {
+                        ControlRequest::Build {
+                            session_id,
+                            dataflow,
+                            git_sources,
+                            prev_git_sources,
+                            local_working_dir,
+                            uv,
+                        } => {
+                            // assign a random build id
+                            let build_id = BuildId::generate();
+
+                            let result = build_dataflow(
+                                build_id,
+                                session_id,
+                                dataflow,
+                                git_sources,
+                                prev_git_sources,
+                                local_working_dir,
+                                &clock,
+                                uv,
+                                &mut daemon_connections,
+                            )
+                            .await;
+                            match result {
+                                Ok(build) => {
+                                    running_builds.insert(build_id, build);
+                                    let _ = reply_sender.send(Ok(
+                                        ControlRequestReply::DataflowBuildTriggered { build_id },
+                                    ));
+                                }
+                                Err(err) => {
+                                    let _ = reply_sender.send(Err(err));
+                                }
+                            }
+                        }
+                        ControlRequest::WaitForBuild { build_id } => {
+                            if let Some(build) = running_builds.get_mut(&build_id) {
+                                build.build_result.register(reply_sender);
+                            } else if let Some(result) = finished_builds.get_mut(&build_id) {
+                                result.register(reply_sender);
+                            } else {
+                                let _ =
+                                    reply_sender.send(Err(eyre!("unknown build id {build_id}")));
+                            }
+                        }
                         ControlRequest::Start {
+                            build_id,
+                            session_id,
                             dataflow,
                             name,
                             local_working_dir,
@@ -408,6 +478,8 @@ async fn start_inner(
                                     }
                                 }
                                 let dataflow = start_dataflow(
+                                    build_id,
+                                    session_id,
                                     dataflow,
                                     local_working_dir,
                                     name,
@@ -418,16 +490,30 @@ async fn start_inner(
                                 .await?;
                                 Ok(dataflow)
                             };
-                            let reply = inner.await.map(|dataflow| {
-                                let uuid = dataflow.uuid;
-                                running_dataflows.insert(uuid, dataflow);
-                                ControlRequestReply::DataflowStarted { uuid }
-                            });
-                            let _ = reply_sender.send(reply);
+                            match inner.await {
+                                Ok(dataflow) => {
+                                    let uuid = dataflow.uuid;
+                                    running_dataflows.insert(uuid, dataflow);
+                                    let _ = reply_sender.send(Ok(
+                                        ControlRequestReply::DataflowStartTriggered { uuid },
+                                    ));
+                                }
+                                Err(err) => {
+                                    let _ = reply_sender.send(Err(err));
+                                }
+                            }
+                        }
+                        ControlRequest::WaitForSpawn { dataflow_id } => {
+                            if let Some(dataflow) = running_dataflows.get_mut(&dataflow_id) {
+                                dataflow.spawn_result.register(reply_sender);
+                            } else {
+                                let _ =
+                                    reply_sender.send(Err(eyre!("unknown dataflow {dataflow_id}")));
+                            }
                         }
                         ControlRequest::Check { dataflow_uuid } => {
                             let status = match &running_dataflows.get(&dataflow_uuid) {
-                                Some(_) => ControlRequestReply::DataflowStarted {
+                                Some(_) => ControlRequestReply::DataflowSpawned {
                                     uuid: dataflow_uuid,
                                 },
                                 None => ControlRequestReply::DataflowStopped {
@@ -495,7 +581,7 @@ async fn start_inner(
 
                             match dataflow {
                                 Ok(dataflow) => {
-                                    dataflow.reply_senders.push(reply_sender);
+                                    dataflow.stop_reply_senders.push(reply_sender);
                                 }
                                 Err(err) => {
                                     let _ = reply_sender.send(Err(err));
@@ -528,7 +614,7 @@ async fn start_inner(
 
                                 match dataflow {
                                     Ok(dataflow) => {
-                                        dataflow.reply_senders.push(reply_sender);
+                                        dataflow.stop_reply_senders.push(reply_sender);
                                     }
                                     Err(err) => {
                                         let _ = reply_sender.send(Err(err));
@@ -626,6 +712,27 @@ async fn start_inner(
                                 "LogSubscribe request should be handled separately"
                             )));
                         }
+                        ControlRequest::BuildLogSubscribe { .. } => {
+                            let _ = reply_sender.send(Err(eyre::eyre!(
+                                "BuildLogSubscribe request should be handled separately"
+                            )));
+                        }
+                        ControlRequest::CliAndDefaultDaemonOnSameMachine => {
+                            let mut default_daemon_ip = None;
+                            if let Some(default_id) = daemon_connections.unnamed().next() {
+                                if let Some(connection) = daemon_connections.get(default_id) {
+                                    if let Ok(addr) = connection.stream.peer_addr() {
+                                        default_daemon_ip = Some(addr.ip());
+                                    }
+                                }
+                            }
+                            let _ = reply_sender.send(Ok(
+                                ControlRequestReply::CliAndDefaultDaemonIps {
+                                    default_daemon: default_daemon_ip,
+                                    cli: None, // filled later
+                                },
+                            ));
+                        }
                     }
                 }
                 ControlEvent::Error(err) => tracing::error!("{err:?}"),
@@ -640,6 +747,17 @@ async fn start_inner(
                             .push(LogSubscriber::new(level, connection));
                     }
                 }
+                ControlEvent::BuildLogSubscribe {
+                    build_id,
+                    level,
+                    connection,
+                } => {
+                    if let Some(build) = running_builds.get_mut(&build_id) {
+                        build
+                            .log_subscribers
+                            .push(LogSubscriber::new(level, connection));
+                    }
+                }
             },
             Event::DaemonHeartbeatInterval => {
                 let mut disconnected = BTreeSet::new();
@@ -695,14 +813,89 @@ async fn start_inner(
                 }
             }
             Event::Log(message) => {
-                if let Some(dataflow) = running_dataflows.get_mut(&message.dataflow_id) {
-                    send_log_message(dataflow, &message).await;
+                if let Some(dataflow_id) = &message.dataflow_id {
+                    if let Some(dataflow) = running_dataflows.get_mut(dataflow_id) {
+                        send_log_message(&mut dataflow.log_subscribers, &message).await;
+                    }
+                }
+                if let Some(build_id) = message.build_id {
+                    if let Some(build) = running_builds.get_mut(&build_id) {
+                        send_log_message(&mut build.log_subscribers, &message).await;
+                    }
                 }
             }
             Event::DaemonExit { daemon_id } => {
                 tracing::info!("Daemon `{daemon_id}` exited");
                 daemon_connections.remove(&daemon_id);
             }
+            Event::DataflowBuildResult {
+                build_id,
+                daemon_id,
+                result,
+            } => match running_builds.get_mut(&build_id) {
+                Some(build) => {
+                    build.pending_build_results.remove(&daemon_id);
+                    match result {
+                        Ok(()) => {}
+                        Err(err) => {
+                            build.errors.push(format!("{err:?}"));
+                        }
+                    };
+                    if build.pending_build_results.is_empty() {
+                        tracing::info!("dataflow build finished: `{build_id}`");
+                        let mut build = running_builds.remove(&build_id).unwrap();
+                        let result = if build.errors.is_empty() {
+                            Ok(())
+                        } else {
+                            Err(format!("build failed: {}", build.errors.join("\n\n")))
+                        };
+
+                        build.build_result.set_result(Ok(
+                            ControlRequestReply::DataflowBuildFinished { build_id, result },
+                        ));
+
+                        finished_builds.insert(build_id, build.build_result);
+                    }
+                }
+                None => {
+                    tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
+                }
+            },
+            Event::DataflowSpawnResult {
+                dataflow_id,
+                daemon_id,
+                result,
+            } => match running_dataflows.get_mut(&dataflow_id) {
+                Some(dataflow) => {
+                    dataflow.pending_spawn_results.remove(&daemon_id);
+                    match result {
+                        Ok(()) => {
+                            if dataflow.pending_spawn_results.is_empty() {
+                                tracing::info!("successfully spawned dataflow `{dataflow_id}`",);
+                                dataflow.spawn_result.set_result(Ok(
+                                    ControlRequestReply::DataflowSpawned { uuid: dataflow_id },
+                                ));
+                            }
+                        }
+                        Err(err) => {
+                            tracing::warn!("error while spawning dataflow `{dataflow_id}`");
+                            dataflow.spawn_result.set_result(Err(err));
+                        }
+                    };
+                }
+                None => {
+                    tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
+                }
+            },
+        }
+
+        // warn if event handling took too long -> the main loop should never be blocked for too long
+        let elapsed = start.elapsed();
+        if elapsed > Duration::from_millis(100) {
+            tracing::warn!(
+                "Coordinator took {}ms for handling event: {event_kind}",
+                elapsed.as_millis()
+            );
         }
     }
 
@@ -711,8 +904,8 @@ async fn start_inner(
     Ok(())
 }
 
-async fn send_log_message(dataflow: &mut RunningDataflow, message: &LogMessage) {
-    for subscriber in &mut dataflow.log_subscribers {
+async fn send_log_message(log_subscribers: &mut Vec<LogSubscriber>, message: &LogMessage) {
+    for subscriber in log_subscribers.iter_mut() {
         let send_result =
             tokio::time::timeout(Duration::from_millis(100), subscriber.send_message(message));
 
@@ -720,7 +913,7 @@ async fn send_log_message(dataflow: &mut RunningDataflow, message: &LogMessage)
             subscriber.close();
         }
     }
-    dataflow.log_subscribers.retain(|s| !s.is_closed());
+    log_subscribers.retain(|s| !s.is_closed());
 }
 
 fn dataflow_result(
@@ -787,6 +980,15 @@ async fn send_heartbeat_message(
         .wrap_err("failed to send heartbeat message to daemon")
 }
 
+struct RunningBuild {
+    errors: Vec<String>,
+    build_result: CachedResult,
+
+    log_subscribers: Vec<LogSubscriber>,
+
+    pending_build_results: BTreeSet<DaemonId>,
+}
+
 struct RunningDataflow {
     name: Option<String>,
     uuid: Uuid,
@@ -797,9 +999,66 @@ struct RunningDataflow {
     exited_before_subscribe: Vec<NodeId>,
     nodes: BTreeMap<NodeId, ResolvedNode>,
 
-    reply_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
+    spawn_result: CachedResult,
+    stop_reply_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
 
     log_subscribers: Vec<LogSubscriber>,
+
+    pending_spawn_results: BTreeSet<DaemonId>,
+}
+
+pub enum CachedResult {
+    Pending {
+        result_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
+    },
+    Cached {
+        result: eyre::Result<ControlRequestReply>,
+    },
+}
+
+impl Default for CachedResult {
+    fn default() -> Self {
+        Self::Pending {
+            result_senders: Vec::new(),
+        }
+    }
+}
+
+impl CachedResult {
+    fn register(
+        &mut self,
+        reply_sender: tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>,
+    ) {
+        match self {
+            CachedResult::Pending { result_senders } => result_senders.push(reply_sender),
+            CachedResult::Cached { result } => {
+                Self::send_result_to(result, reply_sender);
+            }
+        }
+    }
+
+    fn set_result(&mut self, result: eyre::Result<ControlRequestReply>) {
+        match self {
+            CachedResult::Pending { result_senders } => {
+                for sender in result_senders.drain(..) {
+                    Self::send_result_to(&result, sender);
+                }
+                *self = CachedResult::Cached { result };
+            }
+            CachedResult::Cached { .. } => {}
+        }
+    }
+
+    fn send_result_to(
+        result: &eyre::Result<ControlRequestReply>,
+        sender: oneshot::Sender<eyre::Result<ControlRequestReply>>,
+    ) {
+        let result = match result {
+            Ok(r) => Ok(r.clone()),
+            Err(err) => Err(eyre!("{err:?}")),
+        };
+        let _ = sender.send(result);
+    }
 }
 
 struct ArchivedDataflow {
@@ -943,7 +1202,7 @@ async fn retrieve_logs(
     let machine_ids: Vec<Option<String>> = nodes
         .values()
         .filter(|node| node.id == node_id)
-        .map(|node| node.deploy.machine.clone())
+        .map(|node| node.deploy.as_ref().and_then(|d| d.machine.clone()))
         .collect();
 
     let machine_id = if let [machine_id] = &machine_ids[..] {
@@ -992,9 +1251,127 @@ async fn retrieve_logs(
     reply_logs.map_err(|err| eyre!(err))
 }
 
+#[allow(clippy::too_many_arguments)]
+#[tracing::instrument(skip(daemon_connections, clock))]
+async fn build_dataflow(
+    build_id: BuildId,
+    session_id: SessionId,
+    dataflow: Descriptor,
+    git_sources: BTreeMap<NodeId, GitSource>,
+    prev_git_sources: BTreeMap<NodeId, GitSource>,
+    local_working_dir: Option<PathBuf>,
+    clock: &HLC,
+    uv: bool,
+    daemon_connections: &mut DaemonConnections,
+) -> eyre::Result<RunningBuild> {
+    let nodes = dataflow.resolve_aliases_and_set_defaults()?;
+
+    let mut git_sources_by_daemon = git_sources
+        .into_iter()
+        .into_grouping_map_by(|(id, _)| {
+            nodes
+                .get(id)
+                .and_then(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()))
+        })
+        .collect();
+    let mut prev_git_sources_by_daemon = prev_git_sources
+        .into_iter()
+        .into_grouping_map_by(|(id, _)| {
+            nodes
+                .get(id)
+                .and_then(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()))
+        })
+        .collect();
+
+    let nodes_by_daemon = nodes
+        .values()
+        .into_group_map_by(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()));
+
+    let mut daemons = BTreeSet::new();
+    for (machine, nodes_on_machine) in &nodes_by_daemon {
+        let nodes_on_machine = nodes_on_machine.iter().map(|n| n.id.clone()).collect();
+        tracing::debug!(
+            "Running dataflow build `{build_id}` on machine `{machine:?}` (nodes: {nodes_on_machine:?})"
+        );
+
+        let build_command = BuildDataflowNodes {
+            build_id,
+            session_id,
+            local_working_dir: local_working_dir.clone(),
+            git_sources: git_sources_by_daemon.remove(machine).unwrap_or_default(),
+            prev_git_sources: prev_git_sources_by_daemon
+                .remove(machine)
+                .unwrap_or_default(),
+            dataflow_descriptor: dataflow.clone(),
+            nodes_on_machine,
+            uv,
+        };
+        let message = serde_json::to_vec(&Timestamped {
+            inner: DaemonCoordinatorEvent::Build(build_command),
+            timestamp: clock.new_timestamp(),
+        })?;
+
+        let daemon_id =
+            build_dataflow_on_machine(daemon_connections, machine.map(|s| s.as_str()), &message)
+                .await
+                .wrap_err_with(|| format!("failed to build dataflow on machine `{machine:?}`"))?;
+        daemons.insert(daemon_id);
+    }
+
+    tracing::info!("successfully triggered dataflow build `{build_id}`",);
+
+    Ok(RunningBuild {
+        errors: Vec::new(),
+        build_result: CachedResult::default(),
+        log_subscribers: Vec::new(),
+        pending_build_results: daemons,
+    })
+}
+
+async fn build_dataflow_on_machine(
+    daemon_connections: &mut DaemonConnections,
+    machine: Option<&str>,
+    message: &[u8],
+) -> Result<DaemonId, eyre::ErrReport> {
+    let daemon_id = match machine {
+        Some(machine) => daemon_connections
+            .get_matching_daemon_id(machine)
+            .wrap_err_with(|| format!("no matching daemon for machine id {machine:?}"))?
+            .clone(),
+        None => daemon_connections
+            .unnamed()
+            .next()
+            .wrap_err("no unnamed daemon connections")?
+            .clone(),
+    };
+
+    let daemon_connection = daemon_connections
+        .get_mut(&daemon_id)
+        .wrap_err_with(|| format!("no daemon connection for daemon `{daemon_id}`"))?;
+    tcp_send(&mut daemon_connection.stream, message)
+        .await
+        .wrap_err("failed to send build message to daemon")?;
+
+    let reply_raw = tcp_receive(&mut daemon_connection.stream)
+        .await
+        .wrap_err("failed to receive build reply from daemon")?;
+    match serde_json::from_slice(&reply_raw)
+        .wrap_err("failed to deserialize build reply from daemon")?
+    {
+        DaemonCoordinatorReply::TriggerBuildResult(result) => result
+            .map_err(|e| eyre!(e))
+            .wrap_err("daemon returned an error")?,
+        _ => bail!("unexpected reply"),
+    }
+    Ok(daemon_id)
+}
+
+#[allow(clippy::too_many_arguments)]
 async fn start_dataflow(
+    build_id: Option<BuildId>,
+    session_id: SessionId,
     dataflow: Descriptor,
-    working_dir: PathBuf,
+    local_working_dir: Option<PathBuf>,
     name: Option<String>,
     daemon_connections: &mut DaemonConnections,
     clock: &HLC,
@@ -1004,7 +1381,16 @@ async fn start_dataflow(
         uuid,
         daemons,
         nodes,
-    } = spawn_dataflow(dataflow, working_dir, daemon_connections, clock, uv).await?;
+    } = spawn_dataflow(
+        build_id,
+        session_id,
+        dataflow,
+        local_working_dir,
+        daemon_connections,
+        clock,
+        uv,
+    )
+    .await?;
     Ok(RunningDataflow {
         uuid,
         name,
@@ -1014,10 +1400,12 @@ async fn start_dataflow(
             BTreeSet::new()
         },
         exited_before_subscribe: Default::default(),
-        daemons,
+        daemons: daemons.clone(),
         nodes,
-        reply_senders: Vec::new(),
+        spawn_result: CachedResult::default(),
+        stop_reply_senders: Vec::new(),
         log_subscribers: Vec::new(),
+        pending_spawn_results: daemons,
     })
 }
 
@@ -1092,6 +1480,16 @@ pub enum Event {
     DaemonExit {
         daemon_id: dora_message::common::DaemonId,
     },
+    DataflowBuildResult {
+        build_id: BuildId,
+        daemon_id: DaemonId,
+        result: eyre::Result<()>,
+    },
+    DataflowSpawnResult {
+        dataflow_id: uuid::Uuid,
+        daemon_id: DaemonId,
+        result: eyre::Result<()>,
+    },
 }
 
 impl Event {
@@ -1103,6 +1501,23 @@ impl Event {
             _ => true,
         }
     }
+
+    fn kind(&self) -> &'static str {
+        match self {
+            Event::NewDaemonConnection(_) => "NewDaemonConnection",
+            Event::DaemonConnectError(_) => "DaemonConnectError",
+            Event::DaemonHeartbeat { .. } => "DaemonHeartbeat",
+            Event::Dataflow { .. } => "Dataflow",
+            Event::Control(_) => "Control",
+            Event::Daemon(_) => "Daemon",
+            Event::DaemonHeartbeatInterval => "DaemonHeartbeatInterval",
+            Event::CtrlC => "CtrlC",
+            Event::Log(_) => "Log",
+            Event::DaemonExit { .. } => "DaemonExit",
+            Event::DataflowBuildResult { .. } => "DataflowBuildResult",
+            Event::DataflowSpawnResult { .. } => "DataflowSpawnResult",
+        }
+    }
 }
 
 #[derive(Debug)]

binaries/coordinator/src/listener.rs

@@ -112,6 +112,29 @@ pub async fn handle_connection(
                         break;
                     }
                 }
+                DaemonEvent::BuildResult { build_id, result } => {
+                    let event = Event::DataflowBuildResult {
+                        build_id,
+                        daemon_id,
+                        result: result.map_err(|err| eyre::eyre!(err)),
+                    };
+                    if events_tx.send(event).await.is_err() {
+                        break;
+                    }
+                }
+                DaemonEvent::SpawnResult {
+                    dataflow_id,
+                    result,
+                } => {
+                    let event = Event::DataflowSpawnResult {
+                        dataflow_id,
+                        daemon_id,
+                        result: result.map_err(|err| eyre::eyre!(err)),
+                    };
+                    if events_tx.send(event).await.is_err() {
+                        break;
+                    }
+                }
             },
         };
     }

binaries/coordinator/src/log_subscriber.rs

@@ -17,9 +17,15 @@ impl LogSubscriber {
     }
 
     pub async fn send_message(&mut self, message: &LogMessage) -> eyre::Result<()> {
-        if message.level > self.level {
-            return Ok(());
+        match message.level {
+            dora_core::build::LogLevelOrStdout::LogLevel(level) => {
+                if level > self.level {
+                    return Ok(());
+                }
+            }
+            dora_core::build::LogLevelOrStdout::Stdout => {}
         }
+
         let message = serde_json::to_vec(&message)?;
         let connection = self.connection.as_mut().context("connection is closed")?;
         tcp_send(connection, &message)

binaries/coordinator/src/run/mod.rs

@@ -10,6 +10,7 @@ use dora_message::{
     daemon_to_coordinator::DaemonCoordinatorReply,
     descriptor::{Descriptor, ResolvedNode},
     id::NodeId,
+    BuildId, SessionId,
 };
 use eyre::{bail, eyre, ContextCompat, WrapErr};
 use itertools::Itertools;
@@ -21,8 +22,10 @@ use uuid::{NoContext, Timestamp, Uuid};
 
 #[tracing::instrument(skip(daemon_connections, clock))]
 pub(super) async fn spawn_dataflow(
+    build_id: Option<BuildId>,
+    session_id: SessionId,
     dataflow: Descriptor,
-    working_dir: PathBuf,
+    local_working_dir: Option<PathBuf>,
     daemon_connections: &mut DaemonConnections,
     clock: &HLC,
     uv: bool,
@@ -30,7 +33,9 @@ pub(super) async fn spawn_dataflow(
     let nodes = dataflow.resolve_aliases_and_set_defaults()?;
     let uuid = Uuid::new_v7(Timestamp::now(NoContext));
 
-    let nodes_by_daemon = nodes.values().into_group_map_by(|n| &n.deploy.machine);
+    let nodes_by_daemon = nodes
+        .values()
+        .into_group_map_by(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()));
 
     let mut daemons = BTreeSet::new();
     for (machine, nodes_on_machine) in &nodes_by_daemon {
@@ -40,8 +45,10 @@ pub(super) async fn spawn_dataflow(
         );
 
         let spawn_command = SpawnDataflowNodes {
+            build_id,
+            session_id,
             dataflow_id: uuid,
-            working_dir: working_dir.clone(),
+            local_working_dir: local_working_dir.clone(),
             nodes: nodes.clone(),
             dataflow_descriptor: dataflow.clone(),
             spawn_nodes,
@@ -52,13 +59,14 @@ pub(super) async fn spawn_dataflow(
             timestamp: clock.new_timestamp(),
         })?;
 
-        let daemon_id = spawn_dataflow_on_machine(daemon_connections, machine.as_deref(), &message)
-            .await
-            .wrap_err_with(|| format!("failed to spawn dataflow on machine `{machine:?}`"))?;
+        let daemon_id =
+            spawn_dataflow_on_machine(daemon_connections, machine.map(|m| m.as_str()), &message)
+                .await
+                .wrap_err_with(|| format!("failed to spawn dataflow on machine `{machine:?}`"))?;
         daemons.insert(daemon_id);
     }
 
-    tracing::info!("successfully spawned dataflow `{uuid}`");
+    tracing::info!("successfully triggered dataflow spawn `{uuid}`",);
 
     Ok(SpawnedDataflow {
         uuid,
@@ -90,13 +98,14 @@ async fn spawn_dataflow_on_machine(
     tcp_send(&mut daemon_connection.stream, message)
         .await
         .wrap_err("failed to send spawn message to daemon")?;
+
     let reply_raw = tcp_receive(&mut daemon_connection.stream)
         .await
         .wrap_err("failed to receive spawn reply from daemon")?;
     match serde_json::from_slice(&reply_raw)
         .wrap_err("failed to deserialize spawn reply from daemon")?
     {
-        DaemonCoordinatorReply::SpawnResult(result) => result
+        DaemonCoordinatorReply::TriggerSpawnResult(result) => result
             .map_err(|e| eyre!(e))
             .wrap_err("daemon returned an error")?,
         _ => bail!("unexpected reply"),

binaries/daemon/Cargo.toml

@@ -24,14 +24,14 @@ tracing = "0.1.36"
 tracing-opentelemetry = { version = "0.18.0", optional = true }
 futures-concurrency = "7.1.0"
 serde_json = "1.0.86"
-dora-core = { workspace = true }
+dora-core = { workspace = true, features = ["build"] }
 flume = "0.10.14"
 dora-download = { workspace = true }
 dora-tracing = { workspace = true, optional = true }
 dora-arrow-convert = { workspace = true }
 dora-node-api = { workspace = true }
 dora-message = { workspace = true }
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 uuid = { version = "1.7", features = ["v7"] }
 futures = "0.3.25"
 shared-memory-server = { workspace = true }
@@ -44,3 +44,7 @@ sysinfo = "0.30.11"
 crossbeam = "0.8.4"
 crossbeam-skiplist = "0.1.3"
 zenoh = "1.1.1"
+url = "2.5.4"
+git2 = { workspace = true }
+dunce = "1.0.5"
+itertools = "0.14"

binaries/daemon/src/log.rs

@@ -1,14 +1,21 @@
 use std::{
+    ops::{Deref, DerefMut},
     path::{Path, PathBuf},
     sync::Arc,
 };
 
-use dora_core::{config::NodeId, uhlc};
+use dora_core::{
+    build::{BuildLogger, LogLevelOrStdout},
+    config::NodeId,
+    uhlc,
+};
 use dora_message::{
     common::{DaemonId, LogLevel, LogMessage, Timestamped},
     daemon_to_coordinator::{CoordinatorRequest, DaemonEvent},
+    BuildId,
 };
 use eyre::Context;
+use flume::Sender;
 use tokio::net::TcpStream;
 use uuid::Uuid;
 
@@ -39,11 +46,18 @@ impl NodeLogger<'_> {
             .log(level, Some(self.node_id.clone()), target, message)
             .await
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<NodeLogger<'static>> {
+        Ok(NodeLogger {
+            node_id: self.node_id.clone(),
+            logger: self.logger.try_clone().await?,
+        })
+    }
 }
 
 pub struct DataflowLogger<'a> {
     dataflow_id: Uuid,
-    logger: &'a mut DaemonLogger,
+    logger: CowMut<'a, DaemonLogger>,
 }
 
 impl<'a> DataflowLogger<'a> {
@@ -57,12 +71,12 @@ impl<'a> DataflowLogger<'a> {
     pub fn reborrow(&mut self) -> DataflowLogger {
         DataflowLogger {
             dataflow_id: self.dataflow_id,
-            logger: self.logger,
+            logger: CowMut::Borrowed(&mut self.logger),
         }
     }
 
     pub fn inner(&self) -> &DaemonLogger {
-        self.logger
+        &self.logger
     }
 
     pub async fn log(
@@ -73,9 +87,64 @@ impl<'a> DataflowLogger<'a> {
         message: impl Into<String>,
     ) {
         self.logger
-            .log(level, self.dataflow_id, node_id, target, message)
+            .log(level, Some(self.dataflow_id), node_id, target, message)
             .await
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<DataflowLogger<'static>> {
+        Ok(DataflowLogger {
+            dataflow_id: self.dataflow_id,
+            logger: CowMut::Owned(self.logger.try_clone().await?),
+        })
+    }
+}
+
+pub struct NodeBuildLogger<'a> {
+    build_id: BuildId,
+    node_id: NodeId,
+    logger: CowMut<'a, DaemonLogger>,
+}
+
+impl NodeBuildLogger<'_> {
+    pub async fn log(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String>,
+    ) {
+        self.logger
+            .log_build(
+                self.build_id,
+                level.into(),
+                None,
+                Some(self.node_id.clone()),
+                message,
+            )
+            .await
+    }
+
+    pub async fn try_clone_impl(&self) -> eyre::Result<NodeBuildLogger<'static>> {
+        Ok(NodeBuildLogger {
+            build_id: self.build_id,
+            node_id: self.node_id.clone(),
+            logger: CowMut::Owned(self.logger.try_clone().await?),
+        })
+    }
+}
+
+impl BuildLogger for NodeBuildLogger<'_> {
+    type Clone = NodeBuildLogger<'static>;
+
+    fn log_message(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String> + Send,
+    ) -> impl std::future::Future<Output = ()> + Send {
+        self.log(level, message)
+    }
+
+    fn try_clone(&self) -> impl std::future::Future<Output = eyre::Result<Self::Clone>> + Send {
+        self.try_clone_impl()
+    }
 }
 
 pub struct DaemonLogger {
@@ -87,7 +156,15 @@ impl DaemonLogger {
     pub fn for_dataflow(&mut self, dataflow_id: Uuid) -> DataflowLogger {
         DataflowLogger {
             dataflow_id,
-            logger: self,
+            logger: CowMut::Borrowed(self),
+        }
+    }
+
+    pub fn for_node_build(&mut self, build_id: BuildId, node_id: NodeId) -> NodeBuildLogger {
+        NodeBuildLogger {
+            build_id,
+            node_id,
+            logger: CowMut::Borrowed(self),
         }
     }
 
@@ -98,15 +175,39 @@ impl DaemonLogger {
     pub async fn log(
         &mut self,
         level: LogLevel,
-        dataflow_id: Uuid,
+        dataflow_id: Option<Uuid>,
         node_id: Option<NodeId>,
         target: Option<String>,
         message: impl Into<String>,
     ) {
         let message = LogMessage {
+            build_id: None,
             daemon_id: Some(self.daemon_id.clone()),
             dataflow_id,
             node_id,
+            level: level.into(),
+            target,
+            module_path: None,
+            file: None,
+            line: None,
+            message: message.into(),
+        };
+        self.logger.log(message).await
+    }
+
+    pub async fn log_build(
+        &mut self,
+        build_id: BuildId,
+        level: LogLevelOrStdout,
+        target: Option<String>,
+        node_id: Option<NodeId>,
+        message: impl Into<String>,
+    ) {
+        let message = LogMessage {
+            build_id: Some(build_id),
+            daemon_id: Some(self.daemon_id.clone()),
+            dataflow_id: None,
+            node_id,
             level,
             target,
             module_path: None,
@@ -120,10 +221,17 @@ impl DaemonLogger {
     pub(crate) fn daemon_id(&self) -> &DaemonId {
         &self.daemon_id
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<Self> {
+        Ok(Self {
+            daemon_id: self.daemon_id.clone(),
+            logger: self.logger.try_clone().await?,
+        })
+    }
 }
 
 pub struct Logger {
-    pub(super) coordinator_connection: Option<TcpStream>,
+    pub(super) destination: LogDestination,
     pub(super) daemon_id: DaemonId,
     pub(super) clock: Arc<uhlc::HLC>,
 }
@@ -137,73 +245,179 @@ impl Logger {
     }
 
     pub async fn log(&mut self, message: LogMessage) {
-        if let Some(connection) = &mut self.coordinator_connection {
-            let msg = serde_json::to_vec(&Timestamped {
-                inner: CoordinatorRequest::Event {
-                    daemon_id: self.daemon_id.clone(),
-                    event: DaemonEvent::Log(message.clone()),
-                },
-                timestamp: self.clock.new_timestamp(),
-            })
-            .expect("failed to serialize log message");
-            match socket_stream_send(connection, &msg)
-                .await
-                .wrap_err("failed to send log message to dora-coordinator")
-            {
-                Ok(()) => return,
-                Err(err) => tracing::warn!("{err:?}"),
+        match &mut self.destination {
+            LogDestination::Coordinator {
+                coordinator_connection,
+            } => {
+                let message = Timestamped {
+                    inner: CoordinatorRequest::Event {
+                        daemon_id: self.daemon_id.clone(),
+                        event: DaemonEvent::Log(message.clone()),
+                    },
+                    timestamp: self.clock.new_timestamp(),
+                };
+                Self::log_to_coordinator(message, coordinator_connection).await
             }
-        }
-
-        // log message using tracing if reporting to coordinator is not possible
-        match message.level {
-            LogLevel::Error => {
-                if let Some(node_id) = message.node_id {
-                    tracing::error!("{}/{} errored:", message.dataflow_id.to_string(), node_id);
-                }
-                for line in message.message.lines() {
-                    tracing::error!("   {}", line);
-                }
+            LogDestination::Channel { sender } => {
+                let _ = sender.send_async(message).await;
             }
-            LogLevel::Warn => {
-                if let Some(node_id) = message.node_id {
-                    tracing::warn!("{}/{} warned:", message.dataflow_id.to_string(), node_id);
-                }
-                for line in message.message.lines() {
-                    tracing::warn!("    {}", line);
+            LogDestination::Tracing => {
+                // log message using tracing if reporting to coordinator is not possible
+                match message.level {
+                    LogLevelOrStdout::Stdout => {
+                        tracing::info!(
+                            build_id = ?message.build_id.map(|id| id.to_string()),
+                            dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                            node_id = ?message.node_id.map(|id| id.to_string()),
+                            target = message.target,
+                            module_path = message.module_path,
+                            file = message.file,
+                            line = message.line,
+                            "{}",
+                            Indent(&message.message)
+                        )
+                    }
+                    LogLevelOrStdout::LogLevel(level) => match level {
+                        LogLevel::Error => {
+                            tracing::error!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Warn => {
+                            tracing::warn!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Info => {
+                            tracing::info!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Debug => {
+                            tracing::debug!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        _ => {}
+                    },
                 }
             }
-            LogLevel::Info => {
-                if let Some(node_id) = message.node_id {
-                    tracing::info!("{}/{} info:", message.dataflow_id.to_string(), node_id);
-                }
-
-                for line in message.message.lines() {
-                    tracing::info!("    {}", line);
-                }
-            }
-            _ => {}
         }
     }
 
     pub async fn try_clone(&self) -> eyre::Result<Self> {
-        let coordinator_connection = match &self.coordinator_connection {
-            Some(c) => {
-                let addr = c
+        let destination = match &self.destination {
+            LogDestination::Coordinator {
+                coordinator_connection,
+            } => {
+                let addr = coordinator_connection
                     .peer_addr()
                     .context("failed to get coordinator peer addr")?;
                 let new_connection = TcpStream::connect(addr)
                     .await
                     .context("failed to connect to coordinator during logger clone")?;
-                Some(new_connection)
+                LogDestination::Coordinator {
+                    coordinator_connection: new_connection,
+                }
             }
-            None => None,
+            LogDestination::Channel { sender } => LogDestination::Channel {
+                sender: sender.clone(),
+            },
+            LogDestination::Tracing => LogDestination::Tracing,
         };
 
         Ok(Self {
-            coordinator_connection,
+            destination,
             daemon_id: self.daemon_id.clone(),
             clock: self.clock.clone(),
         })
     }
+
+    async fn log_to_coordinator(
+        message: Timestamped<CoordinatorRequest>,
+        connection: &mut TcpStream,
+    ) {
+        let msg = serde_json::to_vec(&message).expect("failed to serialize log message");
+        match socket_stream_send(connection, &msg)
+            .await
+            .wrap_err("failed to send log message to dora-coordinator")
+        {
+            Ok(()) => return,
+            Err(err) => tracing::warn!("{err:?}"),
+        }
+    }
+}
+
+pub enum LogDestination {
+    Coordinator { coordinator_connection: TcpStream },
+    Channel { sender: Sender<LogMessage> },
+    Tracing,
+}
+
+enum CowMut<'a, T> {
+    Borrowed(&'a mut T),
+    Owned(T),
+}
+
+impl<T> Deref for CowMut<'_, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            CowMut::Borrowed(v) => v,
+            CowMut::Owned(v) => v,
+        }
+    }
+}
+
+impl<T> DerefMut for CowMut<'_, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        match self {
+            CowMut::Borrowed(v) => v,
+            CowMut::Owned(v) => v,
+        }
+    }
+}
+
+struct Indent<'a>(&'a str);
+
+impl std::fmt::Display for Indent<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for line in self.0.lines() {
+            write!(f, "   {}", line)?;
+        }
+        Ok(())
+    }
 }

binaries/daemon/src/pending.rs

@@ -59,6 +59,10 @@ impl PendingNodes {
         self.external_nodes = value;
     }
 
+    pub fn local_nodes_pending(&self) -> bool {
+        !self.local_nodes.is_empty()
+    }
+
     pub async fn handle_node_subscription(
         &mut self,
         node_id: NodeId,

binaries/runtime/Cargo.toml

@@ -21,7 +21,7 @@ eyre = "0.6.8"
 futures = "0.3.21"
 futures-concurrency = "7.1.0"
 libloading = "0.7.3"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 tokio = { version = "1.24.2", features = ["full"] }
 tokio-stream = "0.1.8"
 # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html

binaries/runtime/src/lib.rs

@@ -43,7 +43,8 @@ pub fn main() -> eyre::Result<()> {
             .wrap_err("failed to set up tracing subscriber")?;
     }
 
-    let dataflow_descriptor = config.dataflow_descriptor.clone();
+    let dataflow_descriptor = serde_yaml::from_value(config.dataflow_descriptor.clone())
+        .context("failed to parse dataflow descriptor")?;
 
     let operator_definition = if operators.is_empty() {
         bail!("no operators");
@@ -232,10 +233,10 @@ async fn run(
                     }
                 }
             }
-            RuntimeEvent::Event(Event::Stop) => {
+            RuntimeEvent::Event(Event::Stop(cause)) => {
                 // forward stop event to all operators and close the event channels
                 for (_, channel) in operator_channels.drain() {
-                    let _ = channel.send_async(Event::Stop).await;
+                    let _ = channel.send_async(Event::Stop(cause.clone())).await;
                 }
             }
             RuntimeEvent::Event(Event::Reload {

binaries/runtime/src/operator/shared_lib.rs

@@ -182,7 +182,7 @@ impl<'lib> SharedLibraryOperator<'lib> {
             }
 
             let mut operator_event = match event {
-                Event::Stop => dora_operator_api_types::RawEvent {
+                Event::Stop(_) => dora_operator_api_types::RawEvent {
                     input: None,
                     input_closed: None,
                     stop: true,

examples/c++-arrow-dataflow/run.rs

@@ -112,6 +112,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -136,6 +137,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]

examples/c++-dataflow/.gitignore

@@ -1 +1,2 @@
 *.o
+/build

examples/c++-dataflow/run.rs

@@ -133,6 +133,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -157,6 +158,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]

examples/c++-ros2-dataflow/.gitignore

@@ -1 +1,2 @@
 *.o
+/build

examples/c++-ros2-dataflow/run.rs

@@ -90,6 +90,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]
@@ -154,6 +155,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/c-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -63,6 +64,7 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]
@@ -93,6 +95,8 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-lsynchronization");
         clang.arg("-luser32");
         clang.arg("-lwinspool");
+        clang.arg("-lwinhttp");
+        clang.arg("-lrpcrt4");
 
         clang.arg("-Wl,-nodefaultlib:libcmt");
         clang.arg("-D_DLL");
@@ -107,6 +111,7 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-l").arg("pthread");
         clang.arg("-l").arg("c");
         clang.arg("-l").arg("m");
+        clang.arg("-l").arg("z");
     }
     clang.arg("-L").arg(root.join("target").join("debug"));
     clang
@@ -161,6 +166,8 @@ async fn build_c_operator(root: &Path) -> eyre::Result<()> {
         link.arg("-lsynchronization");
         link.arg("-luser32");
         link.arg("-lwinspool");
+        link.arg("-lwinhttp");
+        link.arg("-lrpcrt4");
 
         link.arg("-Wl,-nodefaultlib:libcmt");
         link.arg("-D_DLL");

examples/camera/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/cmake-dataflow/run.rs

@@ -61,6 +61,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/keyboard/dataflow.yml

@@ -0,0 +1,14 @@
+nodes:
+  - id: keyboard
+    build: pip install -e ../../node-hub/dora-keyboard
+    path: dora-keyboard
+    outputs:
+      - char
+    env:
+      DISPLAY: $DISPLAY
+
+  - id: rerun
+    path: dora-rerun
+    build: pip install -e ../../node-hub/dora-rerun
+    inputs:
+      text_input: keyboard/char

examples/multiple-daemons/node/src/main.rs

@@ -26,7 +26,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => println!("Received manual stop"),
+            Event::Stop(_) => println!("Received stop"),
             other => eprintln!("Received unexpected input: {other:?}"),
         }
     }

examples/multiple-daemons/run.rs

@@ -1,3 +1,4 @@
+use dora_cli::session::DataflowSession;
 use dora_coordinator::{ControlEvent, Event};
 use dora_core::{
     descriptor::{read_as_descriptor, DescriptorExt},
@@ -8,7 +9,7 @@ use dora_message::{
     common::DaemonId,
     coordinator_to_cli::{ControlRequestReply, DataflowIdAndName},
 };
-use dora_tracing::set_up_tracing;
+use dora_tracing::TracingBuilder;
 use eyre::{bail, Context};
 
 use std::{
@@ -29,7 +30,9 @@ use uuid::Uuid;
 
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
-    set_up_tracing("multiple-daemon-runner").wrap_err("failed to set up tracing subscriber")?;
+    TracingBuilder::new("multiple-daemon-runner")
+        .with_stdout("debug")
+        .build()?;
 
     let root = Path::new(env!("CARGO_MANIFEST_DIR"));
     std::env::set_current_dir(root.join(file!()).parent().unwrap())
@@ -47,12 +50,15 @@ async fn main() -> eyre::Result<()> {
         IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
         DORA_COORDINATOR_PORT_CONTROL_DEFAULT,
     );
-    let (_coordinator_port, coordinator) = dora_coordinator::start(
+    let (coordinator_port, coordinator) = dora_coordinator::start(
         coordinator_bind,
         coordinator_control_bind,
         ReceiverStream::new(coordinator_events_rx),
     )
     .await?;
+
+    tracing::info!("coordinator running on {coordinator_port}");
+
     let coordinator_addr = Ipv4Addr::LOCALHOST;
     let daemon_a = run_daemon(coordinator_addr.to_string(), "A");
     let daemon_b = run_daemon(coordinator_addr.to_string(), "B");
@@ -135,12 +141,17 @@ async fn start_dataflow(
         .check(&working_dir)
         .wrap_err("could not validate yaml")?;
 
+    let dataflow_session =
+        DataflowSession::read_session(dataflow).context("failed to read DataflowSession")?;
+
     let (reply_sender, reply) = oneshot::channel();
     coordinator_events_tx
         .send(Event::Control(ControlEvent::IncomingRequest {
             request: ControlRequest::Start {
+                build_id: dataflow_session.build_id,
+                session_id: dataflow_session.session_id,
                 dataflow: dataflow_descriptor,
-                local_working_dir: working_dir,
+                local_working_dir: Some(working_dir),
                 name: None,
                 uv: false,
             },
@@ -149,7 +160,21 @@ async fn start_dataflow(
         .await?;
     let result = reply.await??;
     let uuid = match result {
-        ControlRequestReply::DataflowStarted { uuid } => uuid,
+        ControlRequestReply::DataflowStartTriggered { uuid } => uuid,
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    };
+
+    let (reply_sender, reply) = oneshot::channel();
+    coordinator_events_tx
+        .send(Event::Control(ControlEvent::IncomingRequest {
+            request: ControlRequest::WaitForSpawn { dataflow_id: uuid },
+            reply_sender,
+        }))
+        .await?;
+    let result = reply.await??;
+    let uuid = match result {
+        ControlRequestReply::DataflowSpawned { uuid } => uuid,
         ControlRequestReply::Error(err) => bail!("{err}"),
         other => bail!("unexpected start dataflow reply: {other:?}"),
     };
@@ -215,6 +240,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -227,6 +253,7 @@ async fn run_daemon(coordinator: String, machine_id: &str) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--machine-id")

examples/multiple-daemons/sink/src/main.rs

@@ -24,8 +24,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/openai-server/dataflow-rust.yml

@@ -3,14 +3,14 @@ nodes:
     build: cargo build -p dora-openai-proxy-server --release
     path: ../../target/release/dora-openai-proxy-server
     outputs:
-      - chat_completion_request
+      - text
     inputs:
-      completion_reply: dora-echo/echo
+      text: dora-echo/echo
 
   - id: dora-echo
     build: pip install -e ../../node-hub/dora-echo
     path: dora-echo
     inputs:
-      echo: dora-openai-server/chat_completion_request
+      echo: dora-openai-server/text
     outputs:
       - echo

examples/openai-server/openai_api_client.py

@@ -32,11 +32,69 @@ def test_chat_completion(user_input):
         print(f"Error in chat completion: {e}")
 
 
+def test_chat_completion_image_url(user_input):
+    """TODO: Add docstring."""
+    try:
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What is in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+                            },
+                        },
+                    ],
+                }
+            ],
+        )
+        print("Chat Completion Response:")
+        print(response.choices[0].message.content)
+    except Exception as e:
+        print(f"Error in chat completion: {e}")
+
+
+def test_chat_completion_image_base64(user_input):
+    """TODO: Add docstring."""
+    try:
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What is in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII="
+                            },
+                        },
+                    ],
+                }
+            ],
+        )
+        print("Chat Completion Response:")
+        print(response.choices[0].message.content)
+    except Exception as e:
+        print(f"Error in chat completion: {e}")
+
+
 if __name__ == "__main__":
     print("Testing API endpoints...")
-    test_list_models()
+    # test_list_models()
     print("\n" + "=" * 50 + "\n")
 
     chat_input = input("Enter a message for chat completion: ")
     test_chat_completion(chat_input)
+
+    print("\n" + "=" * 50 + "\n")
+
+    test_chat_completion_image_url(chat_input)
+    print("\n" + "=" * 50 + "\n")
+    test_chat_completion_image_base64(chat_input)
     print("\n" + "=" * 50 + "\n")

examples/openai-server/qwenvl.yml

@@ -0,0 +1,16 @@
+nodes:
+  - id: dora-openai-server
+    build: cargo build -p dora-openai-proxy-server --release
+    path: ../../target/release/dora-openai-proxy-server
+    outputs:
+      - text
+    inputs:
+      text: dora-qwen2.5-vl/text
+
+  - id: dora-qwen2.5-vl
+    build: pip install -e ../../node-hub/dora-qwen2-5-vl
+    path: dora-qwen2-5-vl
+    inputs:
+      text: dora-openai-server/text
+    outputs:
+      - text

examples/python-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-multi-env/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-operator-dataflow/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-ros2-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/rerun-viewer/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/rust-dataflow-git/.gitignore

@@ -0,0 +1,4 @@
+/build
+/git
+
+/dataflow.dora-session.yaml

examples/rust-dataflow-git/README.md

@@ -0,0 +1,7 @@
+# Git-based Rust example
+
+To get started:
+
+```bash
+cargo run --example rust-dataflow-git
+```

examples/rust-dataflow-git/dataflow.yml

@@ -0,0 +1,29 @@
+nodes:
+  - id: rust-node
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-node
+    path: target/debug/rust-dataflow-example-node
+    inputs:
+      tick: dora/timer/millis/10
+    outputs:
+      - random
+
+  - id: rust-status-node
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-status-node
+    path: target/debug/rust-dataflow-example-status-node
+    inputs:
+      tick: dora/timer/millis/100
+      random: rust-node/random
+    outputs:
+      - status
+
+  - id: rust-sink
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-sink
+    path: target/debug/rust-dataflow-example-sink
+    inputs:
+      message: rust-status-node/status

examples/rust-dataflow-git/run.rs

@@ -0,0 +1,53 @@
+use dora_tracing::set_up_tracing;
+use eyre::{bail, Context};
+use std::path::Path;
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+    set_up_tracing("rust-dataflow-runner").wrap_err("failed to set up tracing subscriber")?;
+
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    std::env::set_current_dir(root.join(file!()).parent().unwrap())
+        .wrap_err("failed to set working dir")?;
+
+    let args: Vec<String> = std::env::args().collect();
+    let dataflow = if args.len() > 1 {
+        Path::new(&args[1])
+    } else {
+        Path::new("dataflow.yml")
+    };
+    build_dataflow(dataflow).await?;
+
+    run_dataflow(dataflow).await?;
+
+    Ok(())
+}
+
+async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO").unwrap();
+    let mut cmd = tokio::process::Command::new(&cargo);
+    cmd.arg("run");
+    cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
+    cmd.arg("--").arg("build").arg(dataflow);
+    if !cmd.status().await?.success() {
+        bail!("failed to build dataflow");
+    };
+    Ok(())
+}
+
+async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO").unwrap();
+    let mut cmd = tokio::process::Command::new(&cargo);
+    cmd.arg("run");
+    cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
+    cmd.arg("--")
+        .arg("daemon")
+        .arg("--run-dataflow")
+        .arg(dataflow);
+    if !cmd.status().await?.success() {
+        bail!("failed to run dataflow");
+    };
+    Ok(())
+}

examples/rust-dataflow-url/.gitignore

@@ -0,0 +1 @@
+/build

examples/rust-dataflow-url/run.rs

@@ -23,6 +23,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -35,6 +36,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/rust-dataflow/node/src/main.rs

@@ -26,7 +26,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => println!("Received manual stop"),
+            Event::Stop(_) => println!("Received stop"),
             other => eprintln!("Received unexpected input: {other:?}"),
         }
     }

examples/rust-dataflow/run.rs

@@ -16,7 +16,6 @@ async fn main() -> eyre::Result<()> {
     } else {
         Path::new("dataflow.yml")
     };
-
     build_dataflow(dataflow).await?;
 
     run_dataflow(dataflow).await?;
@@ -29,6 +28,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -41,6 +41,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/rust-dataflow/sink-dynamic/src/main.rs

@@ -25,8 +25,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/rust-dataflow/sink/src/main.rs

@@ -24,8 +24,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/rust-dataflow/status-node/src/main.rs

@@ -29,7 +29,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("ignoring unexpected input {other}"),
             },
-            Event::Stop => {}
+            Event::Stop(_) => {}
             Event::InputClosed { id } => {
                 println!("input `{id}` was closed");
                 if *id == "random" {

examples/rust-ros2-dataflow/node/src/main.rs

@@ -119,7 +119,7 @@ fn main() -> eyre::Result<()> {
                     }
                     other => eprintln!("Ignoring unexpected input `{other}`"),
                 },
-                Event::Stop => println!("Received manual stop"),
+                Event::Stop(_) => println!("Received stop"),
                 other => eprintln!("Received unexpected input: {other:?}"),
             },
             MergedEvent::External(pose) => {

examples/rust-ros2-dataflow/run.rs

@@ -23,6 +23,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -35,6 +36,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/so101/Readme.md

@@ -0,0 +1,60 @@
+## SO101 Arm Control
+
+This example provides gamepad control and leader-follower functionality for the SO-101 robotic arm.
+
+### Install Dependencies
+
+install the required Python packages for rerun visualization (optional):
+
+```bash
+# Install the URDF loader for Rerun visualization
+pip install git+https://github.com/dora-rs/rerun-loader-python-urdf
+```
+
+### Hardware Setup
+
+1. Connect your SO-101 arm(s) to your computer via USB/serial
+2. Note the serial port names (e.g.,for linux `/dev/ttyACM0`, `/dev/ttyACM1`)
+3. Connect your gamepad controller
+4. Update the `PORT` environment variable in the YAML files
+
+#### Single Arm Control (arm_gamepad_control.yml)
+
+Control a single SO-101 arm with gamepad input and visualization:
+
+```bash
+dora build arm.yml
+dora run arm.yml
+```
+
+#### Leader-Follower Mode (leader_follower.yml)
+
+Use one arm as a leader to control another follower arm:
+
+```bash
+dora build leader.yml
+dora run leader.yml
+```
+
+#### Serial Port Configuration
+
+Update the `PORT` environment variable in the YAML files:
+
+```yaml
+env:
+  PORT: /dev/ttyACM0  # Change to your actual port
+```
+
+## Troubleshooting
+
+### Serial Connection Issues
+- Check that the arm is powered on and connected
+- Verify the correct serial port in the YAML configuration
+- Ensure proper permissions: `sudo chmod +x PORT`
+
+### Gamepad Not Detected
+- Verify gamepad is connected and recognized by the system
+- Test with `jstest /dev/input/js0` (Linux)
+
+## Safety Notes
+- Always ensure the arm has sufficient clearance before operation

examples/so101/arm_gamepad_control.yml

@@ -0,0 +1,48 @@
+nodes:
+  - id: so101
+    build: pip install -e ../../node-hub/dora-rustypot
+    path: dora-rustypot
+    inputs:
+      tick: dora/timer/millis/10
+      pose: pytorch_kinematics/cmd_vel
+    outputs:
+      - pose
+    env:
+      PORT: /dev/ttyACM0
+      IDS: 1 2 3 4 5
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "so_arm101_description"
+      END_EFFECTOR_LINK: "gripper"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."
+      POSITION_TOLERANCE: 0.01
+      ROTATION_TOLERANCE: 0.03
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+    env:
+      MAX_LINEAR_SPEED: 0.01
+      MAX_ANGULAR_SPEED: 0.05
+
+  # comment below path if you don't want to visualize the arm in rerun
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_so101_new_calib: so101/pose
+    env:
+      so101_new_calib_urdf: "so_arm101_description"
+      so101_new_calib_transform: "0. 0. 0. 1. 0. 0. 0."

examples/so101/leader_follower.yml

@@ -0,0 +1,33 @@
+nodes:
+  - id: so101
+    build: pip install -e ../../node-hub/dora-rustypot
+    path: dora-rustypot
+    inputs:
+      tick: dora/timer/millis/10
+      pose: leader_interface/pose
+    outputs:
+      - pose
+    env:
+      PORT: /dev/ttyACM0
+      IDS: 1 2 3 4 5 6
+
+  - id: leader_interface
+    build: pip install -e ../../node-hub/dora-rustypot
+    path: dora-rustypot
+    inputs:
+      tick: dora/timer/millis/10
+    outputs:
+      - pose  
+    env:
+      PORT: /dev/ttyACM1
+      IDS: 1 2 3 4 5 6
+
+  # comment below path if you don't want to visualize the arms in rerun 
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_so101_new_calib: so101/pose
+    env:
+      so101_new_calib_urdf: "so_arm101_description"
+      so101_new_calib_transform: "0. 0. 0. 1. 0. 0. 0."

examples/speech-to-speech/README.md

@@ -1,4 +1,4 @@
-# Dora Speech to Text example
+# Dora Speech to Speech example
 
 Make sure to have, dora, pip and cargo installed.
 
@@ -23,6 +23,6 @@ sudo apt-get install espeak
 ```bash
 uv venv --seed -p 3.11
 uv pip install -e ../../apis/python/node --reinstall
-dora build kokoro-dev.yml
-dora run kokoro-dev.yml
+dora build kokoro-dev.yml --uv
+dora run kokoro-dev.yml --uv
 ```

examples/urdf/broken_fanuc.yml

@@ -0,0 +1,30 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_m710ic70: pytorch_kinematics/cmd_vel
+    env:
+      m710ic70_urdf: "fanuc_m710ic_description"
+      m710ic70_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "fanuc_m710ic_description"
+      END_EFFECTOR_LINK: "tool0"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/urdf/broken_poppy.yml

@@ -0,0 +1,30 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_poppy_ergo_jr: pytorch_kinematics/cmd_vel
+    env:
+      poppy_ergo_jr_urdf: "poppy_ergo_jr_description"
+      poppy_ergo_jr_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "poppy_ergo_jr_description"
+      END_EFFECTOR_LINK: "section_5"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/urdf/franka.yml

@@ -0,0 +1,35 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_panda: pytorch_kinematics/cmd_vel
+    env:
+      panda_urdf: "panda_description"
+      panda_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+    env:
+      MAX_LINEAR_SPEED: 0.01
+      MAX_ANGULAR_SPEED: 0.05
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "panda_description"
+      END_EFFECTOR_LINK: "panda_link8"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."
+      POSITION_TOLERANCE: 0.001
+      ROTATION_TOLERANCE: 0.001

examples/urdf/gen3.yml

@@ -0,0 +1,30 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_jaco: pytorch_kinematics/cmd_vel
+    env:
+      jaco_urdf: "gen3_description"
+      jaco_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "gen3_description"
+      END_EFFECTOR_LINK: "j2n6s300_end_effector"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/urdf/kuka.yml

@@ -0,0 +1,33 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_iiwa14_primitive_collision: pytorch_kinematics/cmd_vel
+    env:
+      iiwa14_primitive_collision_urdf: "iiwa14_description"
+      iiwa14_primitive_collision_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+    env:
+      MAX_LINEAR_SPEED: 0.02
+      MAX_ANGULAR_SPEED: 0.10
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "iiwa14_description"
+      END_EFFECTOR_LINK: "iiwa_link_7"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/urdf/piper.yml

@@ -0,0 +1,35 @@
+nodes:
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      jointstate_piper_description: pytorch_kinematics/cmd_vel
+    env:
+      piper_description_urdf: "piper_description"
+      piper_description_transform: "0. 0. 0. 1. 0. 0. 0."
+
+  - id: gamepad
+    build: pip install -e ../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+    env:
+      MAX_LINEAR_SPEED: 0.01
+      MAX_ANGULAR_SPEED: 0.05
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+    outputs:
+      - cmd_vel
+    env:
+      MODEL_NAME: "piper_description"
+      END_EFFECTOR_LINK: "link6"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."
+      POSITION_TOLERANCE: 0.001
+      ROTATION_TOLERANCE: 0.001