Merge branch 'dora-rs:main' into main

.github/workflows/ci.yml

@@ -118,6 +118,9 @@ jobs:
       - name: "Rust Dataflow example"
         timeout-minutes: 30
         run: cargo run --example rust-dataflow
+      - name: "Rust Git Dataflow example"
+        timeout-minutes: 30
+        run: cargo run --example rust-dataflow-git
       - name: "Multiple Daemons example"
         timeout-minutes: 30
         run: cargo run --example multiple-daemons
@@ -189,7 +192,7 @@ jobs:
           # only save caches for `main` branch
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - uses: ros-tooling/setup-ros@v0.6
+      - uses: ros-tooling/setup-ros@v0.7
         with:
           required-ros-distributions: humble
       - run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"'
@@ -209,11 +212,11 @@ jobs:
           source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node &
           source /opt/ros/humble/setup.bash && ros2 run examples_rclcpp_minimal_service service_main &
           cargo run --example rust-ros2-dataflow --features="ros2-examples"
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         if: runner.os != 'Windows'
         with:
           python-version: "3.8"
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         if: runner.os == 'Windows'
         with:
           python-version: "3.10"
@@ -321,7 +324,7 @@ jobs:
           dora stop --name ci-rust-dynamic  --grace-duration 5s
           dora destroy
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           # TODO: Support Python 3.13 when https://github.com/pytorch/pytorch/issues/130249 is fixed
           python-version: "3.12"
@@ -339,35 +342,42 @@ jobs:
           # Test Python template Project
           dora new test_python_project --lang python --internal-create-with-path-dependencies
           cd test_python_project
-          uv venv --seed -p 3.11
+          uv venv --seed -p 3.12
           uv pip install -e ../apis/python/node
-          dora build dataflow.yml --uv
           uv pip install ruff pytest
 
+          echo "Running dora up"
+          dora up
+          echo "Running dora build"
+          dora build dataflow.yml --uv
+
           # Check Compliancy
           uv run ruff check .
           uv run pytest
 
           export OPERATING_MODE=SAVE
-          dora up
+          echo "Running dora list"
           dora list
           dora build dataflow.yml --uv
           echo "Running CI Python Test"
           dora start dataflow.yml --name ci-python-test --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-test  --grace-duration 5s
           dora destroy
+          sleep 5
 
           cd ..
 
           # Run Python Node Example
           echo "Running Python Node Example"
           dora up
-          uv venv --seed -p 3.11
+          uv venv --seed -p 3.12
           uv pip install -e apis/python/node
           dora build examples/python-dataflow/dataflow.yml --uv
           dora start examples/python-dataflow/dataflow.yml --name ci-python --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python  --grace-duration 30s
 
           # Run Python Dynamic Node Example
@@ -376,15 +386,18 @@ jobs:
           dora start examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach --uv
           uv run opencv-plot --name plot
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-dynamic  --grace-duration 30s
 
           # Run Python Operator Example
           echo "Running CI Operator Test"
           dora start examples/python-operator-dataflow/dataflow.yml --name ci-python-operator --detach --uv
           sleep 10
+          echo "Running dora stop"
           dora stop --name ci-python-operator  --grace-duration 30s
 
           dora destroy
+          sleep 5
 
           # Run Python queue latency test
           echo "Running CI Queue Latency Test"

.github/workflows/delete-buildjet-cache.yml

@@ -0,0 +1,18 @@
+name: Manually Delete BuildJet Cache
+on:
+  workflow_dispatch:
+    inputs:
+      cache_key:
+        description: 'BuildJet Cache Key to Delete'
+        required: true
+        type: string
+
+jobs:
+  manually-delete-buildjet-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: buildjet/cache-delete@v1
+        with:
+          cache_key: ${{ inputs.cache_key }}

.github/workflows/pip-release.yml

@@ -66,6 +66,7 @@ jobs:
           args: --release --out dist --zig
           manylinux: manylinux_2_28
           working-directory: ${{ matrix.repository.path }}
+          before-script-linux: sudo apt-get install libatomic1-i386-cross libatomic1-armhf-cross && mkdir -p $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/ && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/i686-unknown-linux-gnu/lib/libatomic.so.1 && ln -s /usr/i686-linux-gnu/lib/libatomic.so.1 /opt/hostedtoolcache/Python/3.8.18/x64/lib/libatomic.so.1 && mkdir -p $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/ && ln -s /usr/arm-linux-gnueabihf/lib/libatomic.so.1 $HOME/.rustup/toolchains/1.84-x86_64-unknown-linux-gnu/lib/rustlib/armv7-unknown-linux-gnueabihf/lib/libatomic.so
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4

.gitignore

@@ -35,7 +35,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-build/
+/build/
 develop-eggs/
 dist/
 downloads/
@@ -180,4 +180,4 @@ out/
 #Miscellaneous
 yolo.yml
 
-~*
+~*

Cargo.toml

@@ -72,6 +72,7 @@ dora-metrics = { version = "0.3.11", path = "libraries/extensions/telemetry/metr
 dora-download = { version = "0.3.11", path = "libraries/extensions/download" }
 shared-memory-server = { version = "0.3.11", path = "libraries/shared-memory-server" }
 communication-layer-request-reply = { version = "0.3.11", path = "libraries/communication-layer/request-reply" }
+dora-cli = { version = "0.3.11", path = "binaries/cli" }
 dora-runtime = { version = "0.3.11", path = "binaries/runtime" }
 dora-daemon = { version = "0.3.11", path = "binaries/daemon" }
 dora-coordinator = { version = "0.3.11", path = "binaries/coordinator" }
@@ -79,7 +80,7 @@ dora-ros2-bridge = { version = "0.3.11", path = "libraries/extensions/ros2-bridg
 dora-ros2-bridge-msg-gen = { version = "0.3.11", path = "libraries/extensions/ros2-bridge/msg-gen" }
 dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" }
 # versioned independently from the other dora crates
-dora-message = { version = "0.4.4", path = "libraries/message" }
+dora-message = { version = "0.5.0-alpha", path = "libraries/message" }
 arrow = { version = "54.2.1" }
 arrow-schema = { version = "54.2.1" }
 arrow-data = { version = "54.2.1" }
@@ -91,6 +92,8 @@ pyo3 = { version = "0.23", features = [
     "multiple-pymethods",
 ] }
 pythonize = "0.23"
+git2 = { version = "0.18.0", features = ["vendored-openssl"] }
+serde_yaml = "0.9.33"
 
 [package]
 name = "dora-examples"
@@ -107,6 +110,7 @@ ros2-examples = []
 [dev-dependencies]
 eyre = "0.6.8"
 tokio = "1.24.2"
+dora-cli = { workspace = true }
 dora-coordinator = { workspace = true }
 dora-core = { workspace = true }
 dora-message = { workspace = true }
@@ -135,6 +139,10 @@ path = "examples/vlm/run.rs"
 name = "rust-dataflow"
 path = "examples/rust-dataflow/run.rs"
 
+[[example]]
+name = "rust-dataflow-git"
+path = "examples/rust-dataflow-git/run.rs"
+
 [[example]]
 name = "rust-ros2-dataflow"
 path = "examples/rust-ros2-dataflow/run.rs"

apis/c++/node/src/lib.rs

@@ -144,7 +144,7 @@ pub struct DoraEvent(Option<Event>);
 fn event_type(event: &DoraEvent) -> ffi::DoraEventType {
     match &event.0 {
         Some(event) => match event {
-            Event::Stop => ffi::DoraEventType::Stop,
+            Event::Stop(_) => ffi::DoraEventType::Stop,
             Event::Input { .. } => ffi::DoraEventType::Input,
             Event::InputClosed { .. } => ffi::DoraEventType::InputClosed,
             Event::Error(_) => ffi::DoraEventType::Error,

apis/c/node/src/lib.rs

@@ -91,7 +91,7 @@ pub unsafe extern "C" fn dora_next_event(context: *mut c_void) -> *mut c_void {
 pub unsafe extern "C" fn read_dora_event_type(event: *const ()) -> EventType {
     let event: &Event = unsafe { &*event.cast() };
     match event {
-        Event::Stop => EventType::Stop,
+        Event::Stop(_) => EventType::Stop,
         Event::Input { .. } => EventType::Input,
         Event::InputClosed { .. } => EventType::InputClosed,
         Event::Error(_) => EventType::Error,

apis/python/node/Cargo.toml

@@ -21,10 +21,10 @@ dora-node-api = { workspace = true }
 dora-operator-api-python = { workspace = true }
 pyo3.workspace = true
 eyre = "0.6"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 flume = "0.10.14"
 dora-runtime = { workspace = true, features = ["tracing", "metrics", "python"] }
-dora-daemon = { workspace = true }
+dora-cli = { workspace = true }
 dora-download = { workspace = true }
 arrow = { workspace = true, features = ["pyarrow"] }
 pythonize = { workspace = true }
@@ -33,6 +33,9 @@ dora-ros2-bridge-python = { workspace = true }
 # pyo3_special_method_derive = "0.4.2"
 tokio = { version = "1.24.2", features = ["rt"] }
 
+[build-dependencies]
+pyo3-build-config = "0.23"
+
 [lib]
 name = "dora"
 crate-type = ["cdylib"]

apis/python/node/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    pyo3_build_config::add_extension_module_link_args();
+}

apis/python/node/pyproject.toml

@@ -22,3 +22,11 @@ extend-select = [
     "D",  # pydocstyle
     "UP",
 ]
+
+[tool.maturin.target.x86_64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"
+
+[tool.maturin.target.aarch64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"

apis/python/node/src/lib.rs

@@ -6,7 +6,6 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use arrow::pyarrow::{FromPyArrow, ToPyArrow};
-use dora_daemon::Daemon;
 use dora_download::download_file;
 use dora_node_api::dora_core::config::NodeId;
 use dora_node_api::dora_core::descriptor::source_is_url;
@@ -231,7 +230,7 @@ impl Node {
     /// :rtype: dict
     pub fn dataflow_descriptor(&mut self, py: Python) -> eyre::Result<PyObject> {
         Ok(
-            pythonize::pythonize(py, &self.node.get_mut().dataflow_descriptor())
+            pythonize::pythonize(py, &self.node.get_mut().dataflow_descriptor()?)
                 .map(|x| x.unbind())?,
         )
     }
@@ -382,19 +381,7 @@ pub fn resolve_dataflow(dataflow: String) -> eyre::Result<PathBuf> {
 #[pyfunction]
 #[pyo3(signature = (dataflow_path, uv=None))]
 pub fn run(dataflow_path: String, uv: Option<bool>) -> eyre::Result<()> {
-    let dataflow_path = resolve_dataflow(dataflow_path).context("could not resolve dataflow")?;
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .context("tokio runtime failed")?;
-    let result = rt.block_on(Daemon::run_dataflow(&dataflow_path, uv.unwrap_or_default()))?;
-    match result.is_ok() {
-        true => Ok(()),
-        false => Err(eyre::eyre!(
-            "Dataflow failed to run with error: {:?}",
-            result.node_results
-        )),
-    }
+    dora_cli::command::run(dataflow_path, uv.unwrap_or_default())
 }
 
 #[pymodule]

apis/python/operator/Cargo.toml

@@ -14,7 +14,7 @@ repository.workspace = true
 dora-node-api = { workspace = true }
 pyo3 = { workspace = true, features = ["eyre", "abi3-py37"] }
 eyre = "0.6"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 flume = "0.10.14"
 arrow = { workspace = true, features = ["pyarrow"] }
 arrow-schema = { workspace = true }

apis/python/operator/src/lib.rs

@@ -6,7 +6,7 @@ use std::{
 use arrow::pyarrow::ToPyArrow;
 use dora_node_api::{
     merged::{MergeExternalSend, MergedEvent},
-    DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter,
+    DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, StopCause,
 };
 use eyre::{Context, Result};
 use futures::{Stream, StreamExt};
@@ -146,7 +146,7 @@ impl PyEvent {
 
     fn ty(event: &Event) -> &str {
         match event {
-            Event::Stop => "STOP",
+            Event::Stop(_) => "STOP",
             Event::Input { .. } => "INPUT",
             Event::InputClosed { .. } => "INPUT_CLOSED",
             Event::Error(_) => "ERROR",
@@ -158,6 +158,11 @@ impl PyEvent {
         match event {
             Event::Input { id, .. } => Some(id),
             Event::InputClosed { id } => Some(id),
+            Event::Stop(cause) => match cause {
+                StopCause::Manual => Some("MANUAL"),
+                StopCause::AllInputsClosed => Some("ALL_INPUTS_CLOSED"),
+                &_ => None,
+            },
             _ => None,
         }
     }

apis/rust/node/Cargo.toml

@@ -17,7 +17,7 @@ dora-core = { workspace = true }
 dora-message = { workspace = true }
 shared-memory-server = { workspace = true }
 eyre = "0.6.7"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 tracing = "0.1.33"
 flume = "0.10.14"
 bincode = "1.3.3"

apis/rust/node/src/event_stream/event.rs

@@ -10,7 +10,7 @@ use shared_memory_extended::{Shmem, ShmemConf};
 #[derive(Debug)]
 #[non_exhaustive]
 pub enum Event {
-    Stop,
+    Stop(StopCause),
     Reload {
         operator_id: Option<OperatorId>,
     },
@@ -25,6 +25,13 @@ pub enum Event {
     Error(String),
 }
 
+#[derive(Debug, Clone)]
+#[non_exhaustive]
+pub enum StopCause {
+    Manual,
+    AllInputsClosed,
+}
+
 pub enum RawData {
     Empty,
     Vec(AVec<u8, ConstAlign<128>>),

apis/rust/node/src/event_stream/mod.rs

@@ -11,7 +11,7 @@ use dora_message::{
     node_to_daemon::{DaemonRequest, Timestamped},
     DataflowId,
 };
-pub use event::{Event, MappedInputData, RawData};
+pub use event::{Event, MappedInputData, RawData, StopCause};
 use futures::{
     future::{select, Either},
     Stream, StreamExt,
@@ -199,7 +199,7 @@ impl EventStream {
     fn convert_event_item(item: EventItem) -> Event {
         match item {
             EventItem::NodeEvent { event, ack_channel } => match event {
-                NodeEvent::Stop => Event::Stop,
+                NodeEvent::Stop => Event::Stop(event::StopCause::Manual),
                 NodeEvent::Reload { operator_id } => Event::Reload { operator_id },
                 NodeEvent::InputClosed { id } => Event::InputClosed { id },
                 NodeEvent::Input { id, metadata, data } => {
@@ -234,13 +234,7 @@ impl EventStream {
                         Err(err) => Event::Error(format!("{err:?}")),
                     }
                 }
-                NodeEvent::AllInputsClosed => {
-                    let err = eyre!(
-                        "received `AllInputsClosed` event, which should be handled by background task"
-                    );
-                    tracing::error!("{err:?}");
-                    Event::Error(err.wrap_err("internal error").to_string())
-                }
+                NodeEvent::AllInputsClosed => Event::Stop(event::StopCause::AllInputsClosed),
             },
 
             EventItem::FatalError(err) => {

apis/rust/node/src/event_stream/thread.rs

@@ -92,6 +92,7 @@ fn event_stream_loop(
     clock: Arc<uhlc::HLC>,
 ) {
     let mut tx = Some(tx);
+    let mut close_tx = false;
     let mut pending_drop_tokens: Vec<(DropToken, flume::Receiver<()>, Instant, u64)> = Vec::new();
     let mut drop_tokens = Vec::new();
 
@@ -135,10 +136,8 @@ fn event_stream_loop(
                     data: Some(data), ..
                 } => data.drop_token(),
                 NodeEvent::AllInputsClosed => {
-                    // close the event stream
-                    tx = None;
-                    // skip this internal event
-                    continue;
+                    close_tx = true;
+                    None
                 }
                 _ => None,
             };
@@ -166,6 +165,10 @@ fn event_stream_loop(
             } else {
                 tracing::warn!("dropping event because event `tx` was already closed: `{inner:?}`");
             }
+
+            if close_tx {
+                tx = None;
+            };
         }
     };
     if let Err(err) = result {

apis/rust/node/src/lib.rs

@@ -20,7 +20,7 @@ pub use dora_message::{
     metadata::{Metadata, MetadataParameters, Parameter},
     DataflowId,
 };
-pub use event_stream::{merged, Event, EventStream, MappedInputData, RawData};
+pub use event_stream::{merged, Event, EventStream, MappedInputData, RawData, StopCause};
 pub use flume::Receiver;
 pub use node::{arrow_utils, DataSample, DoraNode, ZERO_COPY_THRESHOLD};
 

apis/rust/node/src/node/mod.rs

@@ -60,7 +60,7 @@ pub struct DoraNode {
     drop_stream: DropStream,
     cache: VecDeque<ShmemHandle>,
 
-    dataflow_descriptor: Descriptor,
+    dataflow_descriptor: serde_yaml::Result<Descriptor>,
     warned_unknown_output: BTreeSet<DataId>,
     _rt: TokioRuntime,
 }
@@ -158,10 +158,9 @@ impl DoraNode {
             ),
         };
 
-        let id = format!("{}/{}", dataflow_id, node_id);
-
         #[cfg(feature = "metrics")]
         {
+            let id = format!("{}/{}", dataflow_id, node_id);
             let monitor_task = async move {
                 if let Err(e) = run_metrics_monitor(id.clone())
                     .await
@@ -200,7 +199,7 @@ impl DoraNode {
             sent_out_shared_memory: HashMap::new(),
             drop_stream,
             cache: VecDeque::new(),
-            dataflow_descriptor,
+            dataflow_descriptor: serde_yaml::from_value(dataflow_descriptor),
             warned_unknown_output: BTreeSet::new(),
             _rt: rt,
         };
@@ -449,8 +448,15 @@ impl DoraNode {
     /// Returns the full dataflow descriptor that this node is part of.
     ///
     /// This method returns the parsed dataflow YAML file.
-    pub fn dataflow_descriptor(&self) -> &Descriptor {
-        &self.dataflow_descriptor
+    pub fn dataflow_descriptor(&self) -> eyre::Result<&Descriptor> {
+        match &self.dataflow_descriptor {
+            Ok(d) => Ok(d),
+            Err(err) => eyre::bail!(
+                "failed to parse dataflow descriptor: {err}\n\n
+                This might be caused by mismatched version numbers of dora \
+                daemon and the dora node API"
+            ),
+        }
     }
 }
 

binaries/cli/Cargo.toml

@@ -27,7 +27,7 @@ dora-node-api-c = { workspace = true }
 dora-operator-api-c = { workspace = true }
 dora-download = { workspace = true }
 serde = { version = "1.0.136", features = ["derive"] }
-serde_yaml = "0.9.11"
+serde_yaml = { workspace = true }
 webbrowser = "0.8.3"
 serde_json = "1.0.86"
 termcolor = "1.1.3"
@@ -37,6 +37,7 @@ communication-layer-request-reply = { workspace = true }
 notify = "5.1.0"
 ctrlc = "3.2.5"
 tracing = "0.1.36"
+tracing-log = "0.2.0"
 dora-tracing = { workspace = true, optional = true }
 bat = "0.24.0"
 dora-daemon = { workspace = true }
@@ -50,7 +51,7 @@ tabwriter = "1.4.0"
 log = { version = "0.4.21", features = ["serde"] }
 colored = "2.1.0"
 env_logger = "0.11.3"
-self_update = { version = "0.27.0", features = [
+self_update = { version = "0.42.0", features = [
     "rustls",
     "archive-zip",
     "archive-tar",
@@ -61,7 +62,11 @@ pyo3 = { workspace = true, features = [
     "abi3",
 ], optional = true }
 self-replace = "1.5.0"
+dunce = "1.0.5"
+git2 = { workspace = true }
 
+[build-dependencies]
+pyo3-build-config = "0.23"
 
 [lib]
 name = "dora_cli"

binaries/cli/build.rs

@@ -1,4 +1,5 @@
 fn main() {
+    pyo3_build_config::add_extension_module_link_args();
     println!(
         "cargo:rustc-env=TARGET={}",
         std::env::var("TARGET").unwrap()

binaries/cli/pyproject.toml

@@ -15,6 +15,14 @@ features = ["python", "pyo3/extension-module"]
 
 [tool.ruff.lint]
 extend-select = [
-  "D",   # pydocstyle
-  "UP"
+  "D",  # pydocstyle
+  "UP",
 ]
+
+[tool.maturin.target.x86_64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"
+
+[tool.maturin.target.aarch64-apple-darwin]
+# macOS deployment target SDK version
+macos-deployment-target = "14.5"

binaries/cli/src/build.rs

@@ -1,116 +0,0 @@
-use dora_core::{
-    config::OperatorId,
-    descriptor::{Descriptor, DescriptorExt, NodeExt, SINGLE_OPERATOR_DEFAULT_ID},
-};
-use dora_message::descriptor::EnvValue;
-use eyre::{eyre, Context};
-use std::{collections::BTreeMap, path::Path, process::Command};
-
-use crate::resolve_dataflow;
-
-pub fn build(dataflow: String, uv: bool) -> eyre::Result<()> {
-    let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-    let descriptor = Descriptor::blocking_read(&dataflow)?;
-    let dataflow_absolute = if dataflow.is_relative() {
-        std::env::current_dir().unwrap().join(dataflow)
-    } else {
-        dataflow.to_owned()
-    };
-    let working_dir = dataflow_absolute.parent().unwrap();
-
-    let default_op_id = OperatorId::from(SINGLE_OPERATOR_DEFAULT_ID.to_string());
-
-    for node in descriptor.nodes {
-        match node.kind()? {
-            dora_core::descriptor::NodeKind::Standard(_) => {
-                run_build_command(node.build.as_deref(), working_dir, uv, node.env.clone())
-                    .with_context(|| {
-                        format!("build command failed for standard node `{}`", node.id)
-                    })?
-            }
-            dora_core::descriptor::NodeKind::Runtime(runtime_node) => {
-                for operator in &runtime_node.operators {
-                    run_build_command(
-                        operator.config.build.as_deref(),
-                        working_dir,
-                        uv,
-                        node.env.clone(),
-                    )
-                    .with_context(|| {
-                        format!(
-                            "build command failed for operator `{}/{}`",
-                            node.id, operator.id
-                        )
-                    })?;
-                }
-            }
-            dora_core::descriptor::NodeKind::Custom(custom_node) => run_build_command(
-                custom_node.build.as_deref(),
-                working_dir,
-                uv,
-                node.env.clone(),
-            )
-            .with_context(|| format!("build command failed for custom node `{}`", node.id))?,
-            dora_core::descriptor::NodeKind::Operator(operator) => run_build_command(
-                operator.config.build.as_deref(),
-                working_dir,
-                uv,
-                node.env.clone(),
-            )
-            .with_context(|| {
-                format!(
-                    "build command failed for operator `{}/{}`",
-                    node.id,
-                    operator.id.as_ref().unwrap_or(&default_op_id)
-                )
-            })?,
-        }
-    }
-
-    Ok(())
-}
-
-fn run_build_command(
-    build: Option<&str>,
-    working_dir: &Path,
-    uv: bool,
-    envs: Option<BTreeMap<String, EnvValue>>,
-) -> eyre::Result<()> {
-    if let Some(build) = build {
-        let lines = build.lines().collect::<Vec<_>>();
-        for build_line in lines {
-            let mut split = build_line.split_whitespace();
-
-            let program = split
-                .next()
-                .ok_or_else(|| eyre!("build command is empty"))?;
-            let mut cmd = if uv && (program == "pip" || program == "pip3") {
-                let mut cmd = Command::new("uv");
-                cmd.arg("pip");
-                cmd
-            } else {
-                Command::new(program)
-            };
-            cmd.args(split);
-
-            // Inject Environment Variables
-            if let Some(envs) = envs.clone() {
-                for (key, value) in envs {
-                    let value = value.to_string();
-                    cmd.env(key, value);
-                }
-            }
-
-            cmd.current_dir(working_dir);
-            let exit_status = cmd
-                .status()
-                .wrap_err_with(|| format!("failed to run `{}`", build))?;
-            if !exit_status.success() {
-                return Err(eyre!("build command `{build_line}` returned {exit_status}"));
-            }
-        }
-        Ok(())
-    } else {
-        Ok(())
-    }
-}

binaries/cli/src/command/build/distributed.rs

@@ -0,0 +1,107 @@
+use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
+use dora_core::descriptor::Descriptor;
+use dora_message::{
+    cli_to_coordinator::ControlRequest,
+    common::{GitSource, LogMessage},
+    coordinator_to_cli::ControlRequestReply,
+    id::NodeId,
+    BuildId,
+};
+use eyre::{bail, Context};
+use std::{
+    collections::BTreeMap,
+    net::{SocketAddr, TcpStream},
+};
+
+use crate::{output::print_log_message, session::DataflowSession};
+
+pub fn build_distributed_dataflow(
+    session: &mut TcpRequestReplyConnection,
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    local_working_dir: Option<std::path::PathBuf>,
+    uv: bool,
+) -> eyre::Result<BuildId> {
+    let build_id = {
+        let reply_raw = session
+            .request(
+                &serde_json::to_vec(&ControlRequest::Build {
+                    session_id: dataflow_session.session_id,
+                    dataflow,
+                    git_sources: git_sources.clone(),
+                    prev_git_sources: dataflow_session.git_sources.clone(),
+                    local_working_dir,
+                    uv,
+                })
+                .unwrap(),
+            )
+            .wrap_err("failed to send start dataflow message")?;
+
+        let result: ControlRequestReply =
+            serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+        match result {
+            ControlRequestReply::DataflowBuildTriggered { build_id } => {
+                eprintln!("dataflow build triggered: {build_id}");
+                build_id
+            }
+            ControlRequestReply::Error(err) => bail!("{err}"),
+            other => bail!("unexpected start dataflow reply: {other:?}"),
+        }
+    };
+    Ok(build_id)
+}
+
+pub fn wait_until_dataflow_built(
+    build_id: BuildId,
+    session: &mut TcpRequestReplyConnection,
+    coordinator_socket: SocketAddr,
+    log_level: log::LevelFilter,
+) -> eyre::Result<BuildId> {
+    // subscribe to log messages
+    let mut log_session = TcpConnection {
+        stream: TcpStream::connect(coordinator_socket)
+            .wrap_err("failed to connect to dora coordinator")?,
+    };
+    log_session
+        .send(
+            &serde_json::to_vec(&ControlRequest::BuildLogSubscribe {
+                build_id,
+                level: log_level,
+            })
+            .wrap_err("failed to serialize message")?,
+        )
+        .wrap_err("failed to send build log subscribe request to coordinator")?;
+    std::thread::spawn(move || {
+        while let Ok(raw) = log_session.receive() {
+            let parsed: eyre::Result<LogMessage> =
+                serde_json::from_slice(&raw).context("failed to parse log message");
+            match parsed {
+                Ok(log_message) => {
+                    print_log_message(log_message, false, true);
+                }
+                Err(err) => {
+                    tracing::warn!("failed to parse log message: {err:?}")
+                }
+            }
+        }
+    });
+
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::WaitForBuild { build_id }).unwrap())
+        .wrap_err("failed to send WaitForBuild message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::DataflowBuildFinished { build_id, result } => match result {
+            Ok(()) => {
+                eprintln!("dataflow build finished successfully");
+                Ok(build_id)
+            }
+            Err(err) => bail!("{err}"),
+        },
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+}

binaries/cli/src/command/build/git.rs

@@ -0,0 +1,45 @@
+use dora_message::{common::GitSource, descriptor::GitRepoRev};
+use eyre::Context;
+
+pub fn fetch_commit_hash(repo_url: String, rev: Option<GitRepoRev>) -> eyre::Result<GitSource> {
+    let mut remote = git2::Remote::create_detached(repo_url.as_bytes())
+        .with_context(|| format!("failed to create git remote for {repo_url}"))?;
+    let connection = remote
+        .connect_auth(git2::Direction::Fetch, None, None)
+        .with_context(|| format!("failed to open connection to {repo_url}"))?;
+    let references = connection
+        .list()
+        .with_context(|| format!("failed to list git references of {repo_url}"))?;
+
+    let expected_name = match &rev {
+        Some(GitRepoRev::Branch(branch)) => format!("refs/heads/{branch}"),
+        Some(GitRepoRev::Tag(tag)) => format!("refs/tags/{tag}"),
+        Some(GitRepoRev::Rev(rev)) => rev.clone(),
+        None => "HEAD".into(),
+    };
+
+    let mut commit_hash = None;
+    for head in references {
+        if head.name() == expected_name {
+            commit_hash = Some(head.oid().to_string());
+            break;
+        }
+    }
+
+    if commit_hash.is_none() {
+        if let Some(GitRepoRev::Rev(rev)) = &rev {
+            // rev might be a commit hash instead of a reference
+            if rev.is_ascii() && rev.bytes().all(|b| b.is_ascii_alphanumeric()) {
+                commit_hash = Some(rev.clone());
+            }
+        }
+    }
+
+    match commit_hash {
+        Some(commit_hash) => Ok(GitSource {
+            repo: repo_url,
+            commit_hash,
+        }),
+        None => eyre::bail!("no matching commit for `{rev:?}`"),
+    }
+}

binaries/cli/src/command/build/local.rs

@@ -0,0 +1,121 @@
+use std::{collections::BTreeMap, path::PathBuf};
+
+use colored::Colorize;
+use dora_core::{
+    build::{BuildInfo, BuildLogger, Builder, GitManager, LogLevelOrStdout, PrevGitSource},
+    descriptor::{Descriptor, DescriptorExt},
+};
+use dora_message::{common::GitSource, id::NodeId};
+use eyre::Context;
+
+use crate::session::DataflowSession;
+
+pub fn build_dataflow_locally(
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    working_dir: PathBuf,
+    uv: bool,
+) -> eyre::Result<BuildInfo> {
+    let runtime = tokio::runtime::Runtime::new()?;
+
+    runtime.block_on(build_dataflow(
+        dataflow,
+        git_sources,
+        dataflow_session,
+        working_dir,
+        uv,
+    ))
+}
+
+async fn build_dataflow(
+    dataflow: Descriptor,
+    git_sources: &BTreeMap<NodeId, GitSource>,
+    dataflow_session: &DataflowSession,
+    base_working_dir: PathBuf,
+    uv: bool,
+) -> eyre::Result<BuildInfo> {
+    let builder = Builder {
+        session_id: dataflow_session.session_id,
+        base_working_dir,
+        uv,
+    };
+    let nodes = dataflow.resolve_aliases_and_set_defaults()?;
+
+    let mut git_manager = GitManager::default();
+    let prev_git_sources = &dataflow_session.git_sources;
+
+    let mut tasks = Vec::new();
+
+    // build nodes
+    for node in nodes.into_values() {
+        let node_id = node.id.clone();
+        let git_source = git_sources.get(&node_id).cloned();
+        let prev_git_source = prev_git_sources.get(&node_id).cloned();
+        let prev_git = prev_git_source.map(|prev_source| PrevGitSource {
+            still_needed_for_this_build: git_sources.values().any(|s| s == &prev_source),
+            git_source: prev_source,
+        });
+
+        let task = builder
+            .clone()
+            .build_node(
+                node,
+                git_source,
+                prev_git,
+                LocalBuildLogger {
+                    node_id: node_id.clone(),
+                },
+                &mut git_manager,
+            )
+            .await
+            .wrap_err_with(|| format!("failed to build node `{node_id}`"))?;
+        tasks.push((node_id, task));
+    }
+
+    let mut info = BuildInfo {
+        node_working_dirs: Default::default(),
+    };
+    for (node_id, task) in tasks {
+        let node = task
+            .await
+            .with_context(|| format!("failed to build node `{node_id}`"))?;
+        info.node_working_dirs
+            .insert(node_id, node.node_working_dir);
+    }
+    Ok(info)
+}
+
+struct LocalBuildLogger {
+    node_id: NodeId,
+}
+
+impl BuildLogger for LocalBuildLogger {
+    type Clone = Self;
+
+    async fn log_message(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String> + Send,
+    ) {
+        let level = match level.into() {
+            LogLevelOrStdout::LogLevel(level) => match level {
+                log::Level::Error => "ERROR ".red(),
+                log::Level::Warn => "WARN  ".yellow(),
+                log::Level::Info => "INFO  ".green(),
+                log::Level::Debug => "DEBUG ".bright_blue(),
+                log::Level::Trace => "TRACE ".dimmed(),
+            },
+            LogLevelOrStdout::Stdout => "stdout".italic().dimmed(),
+        };
+        let node = self.node_id.to_string().bold().bright_black();
+        let message: String = message.into();
+        println!("{node}: {level}   {message}");
+    }
+
+    async fn try_clone(&self) -> eyre::Result<Self::Clone> {
+        Ok(LocalBuildLogger {
+            node_id: self.node_id.clone(),
+        })
+    }
+}

binaries/cli/src/command/build/mod.rs

@@ -0,0 +1,168 @@
+use communication_layer_request_reply::TcpRequestReplyConnection;
+use dora_core::{
+    descriptor::{CoreNodeKind, CustomNode, Descriptor, DescriptorExt},
+    topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST},
+};
+use dora_message::{descriptor::NodeSource, BuildId};
+use eyre::Context;
+use std::collections::BTreeMap;
+
+use crate::{connect_to_coordinator, resolve_dataflow, session::DataflowSession};
+
+use distributed::{build_distributed_dataflow, wait_until_dataflow_built};
+use local::build_dataflow_locally;
+
+mod distributed;
+mod git;
+mod local;
+
+pub fn build(
+    dataflow: String,
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+    uv: bool,
+    force_local: bool,
+) -> eyre::Result<()> {
+    let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_descriptor =
+        Descriptor::blocking_read(&dataflow_path).wrap_err("Failed to read yaml dataflow")?;
+    let mut dataflow_session =
+        DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
+
+    let mut git_sources = BTreeMap::new();
+    let resolved_nodes = dataflow_descriptor
+        .resolve_aliases_and_set_defaults()
+        .context("failed to resolve nodes")?;
+    for (node_id, node) in resolved_nodes {
+        if let CoreNodeKind::Custom(CustomNode {
+            source: NodeSource::GitBranch { repo, rev },
+            ..
+        }) = node.kind
+        {
+            let source = git::fetch_commit_hash(repo, rev)
+                .with_context(|| format!("failed to find commit hash for `{node_id}`"))?;
+            git_sources.insert(node_id, source);
+        }
+    }
+
+    let session = || connect_to_coordinator_with_defaults(coordinator_addr, coordinator_port);
+
+    let build_kind = if force_local {
+        log::info!("Building locally, as requested through `--force-local`");
+        BuildKind::Local
+    } else if dataflow_descriptor.nodes.iter().all(|n| n.deploy.is_none()) {
+        log::info!("Building locally because dataflow does not contain any `deploy` sections");
+        BuildKind::Local
+    } else if coordinator_addr.is_some() || coordinator_port.is_some() {
+        log::info!("Building through coordinator, using the given coordinator socket information");
+        // explicit coordinator address or port set -> there should be a coordinator running
+        BuildKind::ThroughCoordinator {
+            coordinator_session: session().context("failed to connect to coordinator")?,
+        }
+    } else {
+        match session() {
+            Ok(coordinator_session) => {
+                // we found a local coordinator instance at default port -> use it for building
+                log::info!("Found local dora coordinator instance -> building through coordinator");
+                BuildKind::ThroughCoordinator {
+                    coordinator_session,
+                }
+            }
+            Err(_) => {
+                log::warn!("No dora coordinator instance found -> trying a local build");
+                // no coordinator instance found -> do a local build
+                BuildKind::Local
+            }
+        }
+    };
+
+    match build_kind {
+        BuildKind::Local => {
+            log::info!("running local build");
+            // use dataflow dir as base working dir
+            let local_working_dir = dunce::canonicalize(&dataflow_path)
+                .context("failed to canonicalize dataflow path")?
+                .parent()
+                .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
+                .to_owned();
+            let build_info = build_dataflow_locally(
+                dataflow_descriptor,
+                &git_sources,
+                &dataflow_session,
+                local_working_dir,
+                uv,
+            )?;
+
+            dataflow_session.git_sources = git_sources;
+            // generate a random BuildId and store the associated build info
+            dataflow_session.build_id = Some(BuildId::generate());
+            dataflow_session.local_build = Some(build_info);
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+        }
+        BuildKind::ThroughCoordinator {
+            mut coordinator_session,
+        } => {
+            let local_working_dir = super::local_working_dir(
+                &dataflow_path,
+                &dataflow_descriptor,
+                &mut *coordinator_session,
+            )?;
+            let build_id = build_distributed_dataflow(
+                &mut *coordinator_session,
+                dataflow_descriptor,
+                &git_sources,
+                &dataflow_session,
+                local_working_dir,
+                uv,
+            )?;
+
+            dataflow_session.git_sources = git_sources;
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+
+            // wait until dataflow build is finished
+
+            wait_until_dataflow_built(
+                build_id,
+                &mut *coordinator_session,
+                coordinator_socket(coordinator_addr, coordinator_port),
+                log::LevelFilter::Info,
+            )?;
+
+            dataflow_session.build_id = Some(build_id);
+            dataflow_session.local_build = None;
+            dataflow_session
+                .write_out_for_dataflow(&dataflow_path)
+                .context("failed to write out dataflow session file")?;
+        }
+    };
+
+    Ok(())
+}
+
+enum BuildKind {
+    Local,
+    ThroughCoordinator {
+        coordinator_session: Box<TcpRequestReplyConnection>,
+    },
+}
+
+fn connect_to_coordinator_with_defaults(
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+) -> std::io::Result<Box<TcpRequestReplyConnection>> {
+    let coordinator_socket = coordinator_socket(coordinator_addr, coordinator_port);
+    connect_to_coordinator(coordinator_socket)
+}
+
+fn coordinator_socket(
+    coordinator_addr: Option<std::net::IpAddr>,
+    coordinator_port: Option<u16>,
+) -> std::net::SocketAddr {
+    let coordinator_addr = coordinator_addr.unwrap_or(LOCALHOST);
+    let coordinator_port = coordinator_port.unwrap_or(DORA_COORDINATOR_PORT_CONTROL_DEFAULT);
+    (coordinator_addr, coordinator_port).into()
+}

binaries/cli/src/command/mod.rs

@@ -0,0 +1,60 @@
+pub use build::build;
+pub use logs::logs;
+pub use run::run;
+pub use start::start;
+
+use std::path::{Path, PathBuf};
+
+use communication_layer_request_reply::TcpRequestReplyConnection;
+use dora_core::descriptor::Descriptor;
+use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
+use eyre::{bail, Context, ContextCompat};
+
+mod build;
+pub mod check;
+mod logs;
+mod run;
+mod start;
+pub mod up;
+
+fn local_working_dir(
+    dataflow_path: &Path,
+    dataflow_descriptor: &Descriptor,
+    coordinator_session: &mut TcpRequestReplyConnection,
+) -> eyre::Result<Option<PathBuf>> {
+    Ok(
+        if dataflow_descriptor
+            .nodes
+            .iter()
+            .all(|n| n.deploy.as_ref().map(|d| d.machine.as_ref()).is_none())
+            && cli_and_daemon_on_same_machine(coordinator_session)?
+        {
+            Some(
+                dunce::canonicalize(dataflow_path)
+                    .context("failed to canonicalize dataflow file path")?
+                    .parent()
+                    .context("dataflow path has no parent dir")?
+                    .to_owned(),
+            )
+        } else {
+            None
+        },
+    )
+}
+
+fn cli_and_daemon_on_same_machine(session: &mut TcpRequestReplyConnection) -> eyre::Result<bool> {
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::CliAndDefaultDaemonOnSameMachine).unwrap())
+        .wrap_err("failed to send start dataflow message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::CliAndDefaultDaemonIps {
+            default_daemon,
+            cli,
+        } => Ok(default_daemon.is_some() && default_daemon == cli),
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+}

binaries/cli/src/command/run.rs

@@ -0,0 +1,34 @@
+use dora_daemon::{flume, Daemon, LogDestination};
+use eyre::Context;
+use tokio::runtime::Builder;
+
+use crate::{
+    handle_dataflow_result, output::print_log_message, resolve_dataflow, session::DataflowSession,
+};
+
+pub fn run(dataflow: String, uv: bool) -> Result<(), eyre::Error> {
+    let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_session =
+        DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
+    let rt = Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .context("tokio runtime failed")?;
+
+    let (log_tx, log_rx) = flume::bounded(100);
+    std::thread::spawn(move || {
+        for message in log_rx {
+            print_log_message(message, false, false);
+        }
+    });
+
+    let result = rt.block_on(Daemon::run_dataflow(
+        &dataflow_path,
+        dataflow_session.build_id,
+        dataflow_session.local_build,
+        dataflow_session.session_id,
+        uv,
+        LogDestination::Channel { sender: log_tx },
+    ))?;
+    handle_dataflow_result(result, None)
+}

binaries/cli/src/attach.rs → binaries/cli/src/command/start/attach.rs

@@ -1,4 +1,3 @@
-use colored::Colorize;
 use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
 use dora_core::descriptor::{resolve_path, CoreNodeKind, Descriptor, DescriptorExt};
 use dora_message::cli_to_coordinator::ControlRequest;
@@ -16,6 +15,7 @@ use tracing::{error, info};
 use uuid::Uuid;
 
 use crate::handle_dataflow_result;
+use crate::output::print_log_message;
 
 pub fn attach_dataflow(
     dataflow: Descriptor,
@@ -33,6 +33,8 @@ pub fn attach_dataflow(
 
     let nodes = dataflow.resolve_aliases_and_set_defaults()?;
 
+    let print_daemon_name = nodes.values().any(|n| n.deploy.is_some());
+
     let working_dir = dataflow_path
         .canonicalize()
         .context("failed to canonicalize dataflow path")?
@@ -155,39 +157,7 @@ pub fn attach_dataflow(
             },
             Ok(AttachEvent::Control(control_request)) => control_request,
             Ok(AttachEvent::Log(Ok(log_message))) => {
-                let LogMessage {
-                    dataflow_id,
-                    node_id,
-                    daemon_id,
-                    level,
-                    target,
-                    module_path: _,
-                    file: _,
-                    line: _,
-                    message,
-                } = log_message;
-                let level = match level {
-                    log::Level::Error => "ERROR".red(),
-                    log::Level::Warn => "WARN ".yellow(),
-                    log::Level::Info => "INFO ".green(),
-                    other => format!("{other:5}").normal(),
-                };
-                let dataflow = format!(" dataflow `{dataflow_id}`").cyan();
-                let daemon = match daemon_id {
-                    Some(id) => format!(" on daemon `{id}`"),
-                    None => " on default daemon".to_string(),
-                }
-                .bright_black();
-                let node = match node_id {
-                    Some(node_id) => format!(" {node_id}").bold(),
-                    None => "".normal(),
-                };
-                let target = match target {
-                    Some(target) => format!(" {target}").dimmed(),
-                    None => "".normal(),
-                };
-
-                println!("{level}{dataflow}{daemon}{node}{target}: {message}");
+                print_log_message(log_message, false, print_daemon_name);
                 continue;
             }
             Ok(AttachEvent::Log(Err(err))) => {
@@ -202,7 +172,7 @@ pub fn attach_dataflow(
         let result: ControlRequestReply =
             serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
         match result {
-            ControlRequestReply::DataflowStarted { uuid: _ } => (),
+            ControlRequestReply::DataflowSpawned { uuid: _ } => (),
             ControlRequestReply::DataflowStopped { uuid, result } => {
                 info!("dataflow {uuid} stopped");
                 break handle_dataflow_result(result, Some(uuid));

binaries/cli/src/command/start/mod.rs

@@ -0,0 +1,170 @@
+use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
+use dora_core::descriptor::{Descriptor, DescriptorExt};
+use dora_message::{
+    cli_to_coordinator::ControlRequest, common::LogMessage, coordinator_to_cli::ControlRequestReply,
+};
+use eyre::{bail, Context};
+use std::{
+    net::{SocketAddr, TcpStream},
+    path::PathBuf,
+};
+use uuid::Uuid;
+
+use crate::{
+    connect_to_coordinator, output::print_log_message, resolve_dataflow, session::DataflowSession,
+};
+use attach::attach_dataflow;
+
+mod attach;
+
+pub fn start(
+    dataflow: String,
+    name: Option<String>,
+    coordinator_socket: SocketAddr,
+    attach: bool,
+    detach: bool,
+    hot_reload: bool,
+    uv: bool,
+) -> eyre::Result<()> {
+    let (dataflow, dataflow_descriptor, mut session, dataflow_id) =
+        start_dataflow(dataflow, name, coordinator_socket, uv)?;
+
+    let attach = match (attach, detach) {
+        (true, true) => eyre::bail!("both `--attach` and `--detach` are given"),
+        (true, false) => true,
+        (false, true) => false,
+        (false, false) => {
+            println!("attaching to dataflow (use `--detach` to run in background)");
+            true
+        }
+    };
+
+    if attach {
+        let log_level = env_logger::Builder::new()
+            .filter_level(log::LevelFilter::Info)
+            .parse_default_env()
+            .build()
+            .filter();
+
+        attach_dataflow(
+            dataflow_descriptor,
+            dataflow,
+            dataflow_id,
+            &mut *session,
+            hot_reload,
+            coordinator_socket,
+            log_level,
+        )
+    } else {
+        let print_daemon_name = dataflow_descriptor.nodes.iter().any(|n| n.deploy.is_some());
+        // wait until dataflow is started
+        wait_until_dataflow_started(
+            dataflow_id,
+            &mut session,
+            coordinator_socket,
+            log::LevelFilter::Info,
+            print_daemon_name,
+        )
+    }
+}
+
+fn start_dataflow(
+    dataflow: String,
+    name: Option<String>,
+    coordinator_socket: SocketAddr,
+    uv: bool,
+) -> Result<(PathBuf, Descriptor, Box<TcpRequestReplyConnection>, Uuid), eyre::Error> {
+    let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let dataflow_descriptor =
+        Descriptor::blocking_read(&dataflow).wrap_err("Failed to read yaml dataflow")?;
+    let dataflow_session =
+        DataflowSession::read_session(&dataflow).context("failed to read DataflowSession")?;
+
+    let mut session = connect_to_coordinator(coordinator_socket)
+        .wrap_err("failed to connect to dora coordinator")?;
+
+    let local_working_dir =
+        super::local_working_dir(&dataflow, &dataflow_descriptor, &mut *session)?;
+
+    let dataflow_id = {
+        let dataflow = dataflow_descriptor.clone();
+        let session: &mut TcpRequestReplyConnection = &mut *session;
+        let reply_raw = session
+            .request(
+                &serde_json::to_vec(&ControlRequest::Start {
+                    build_id: dataflow_session.build_id,
+                    session_id: dataflow_session.session_id,
+                    dataflow,
+                    name,
+                    local_working_dir,
+                    uv,
+                })
+                .unwrap(),
+            )
+            .wrap_err("failed to send start dataflow message")?;
+
+        let result: ControlRequestReply =
+            serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+        match result {
+            ControlRequestReply::DataflowStartTriggered { uuid } => {
+                eprintln!("dataflow start triggered: {uuid}");
+                uuid
+            }
+            ControlRequestReply::Error(err) => bail!("{err}"),
+            other => bail!("unexpected start dataflow reply: {other:?}"),
+        }
+    };
+    Ok((dataflow, dataflow_descriptor, session, dataflow_id))
+}
+
+fn wait_until_dataflow_started(
+    dataflow_id: Uuid,
+    session: &mut Box<TcpRequestReplyConnection>,
+    coordinator_addr: SocketAddr,
+    log_level: log::LevelFilter,
+    print_daemon_id: bool,
+) -> eyre::Result<()> {
+    // subscribe to log messages
+    let mut log_session = TcpConnection {
+        stream: TcpStream::connect(coordinator_addr)
+            .wrap_err("failed to connect to dora coordinator")?,
+    };
+    log_session
+        .send(
+            &serde_json::to_vec(&ControlRequest::LogSubscribe {
+                dataflow_id,
+                level: log_level,
+            })
+            .wrap_err("failed to serialize message")?,
+        )
+        .wrap_err("failed to send log subscribe request to coordinator")?;
+    std::thread::spawn(move || {
+        while let Ok(raw) = log_session.receive() {
+            let parsed: eyre::Result<LogMessage> =
+                serde_json::from_slice(&raw).context("failed to parse log message");
+            match parsed {
+                Ok(log_message) => {
+                    print_log_message(log_message, false, print_daemon_id);
+                }
+                Err(err) => {
+                    tracing::warn!("failed to parse log message: {err:?}")
+                }
+            }
+        }
+    });
+
+    let reply_raw = session
+        .request(&serde_json::to_vec(&ControlRequest::WaitForSpawn { dataflow_id }).unwrap())
+        .wrap_err("failed to send start dataflow message")?;
+
+    let result: ControlRequestReply =
+        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
+    match result {
+        ControlRequestReply::DataflowSpawned { uuid } => {
+            eprintln!("dataflow started: {uuid}");
+        }
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    }
+    Ok(())
+}

binaries/cli/src/up.rs → binaries/cli/src/command/up.rs

@@ -1,4 +1,4 @@
-use crate::{check::daemon_running, connect_to_coordinator, LOCALHOST};
+use crate::{command::check::daemon_running, connect_to_coordinator, LOCALHOST};
 use dora_core::topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT;
 use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
 use eyre::{bail, Context, ContextCompat};

binaries/cli/src/lib.rs

@@ -1,4 +1,3 @@
-use attach::attach_dataflow;
 use colored::Colorize;
 use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection};
 use dora_coordinator::Event;
@@ -9,7 +8,7 @@ use dora_core::{
         DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT,
     },
 };
-use dora_daemon::Daemon;
+use dora_daemon::{Daemon, LogDestination};
 use dora_download::download_file;
 use dora_message::{
     cli_to_coordinator::ControlRequest,
@@ -31,14 +30,12 @@ use tokio::runtime::Builder;
 use tracing::level_filters::LevelFilter;
 use uuid::Uuid;
 
-mod attach;
-mod build;
-mod check;
+pub mod command;
 mod formatting;
 mod graph;
-mod logs;
+pub mod output;
+pub mod session;
 mod template;
-mod up;
 
 const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
 const LISTEN_WILDCARD: IpAddr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
@@ -82,9 +79,18 @@ enum Command {
         /// Path to the dataflow descriptor file
         #[clap(value_name = "PATH")]
         dataflow: String,
+        /// Address of the dora coordinator
+        #[clap(long, value_name = "IP")]
+        coordinator_addr: Option<IpAddr>,
+        /// Port number of the coordinator control server
+        #[clap(long, value_name = "PORT")]
+        coordinator_port: Option<u16>,
         // Use UV to build nodes.
         #[clap(long, action)]
         uv: bool,
+        // Run build on local machine
+        #[clap(long, action)]
+        local: bool,
     },
     /// Generate a new project or node. Choose the language between Rust, Python, C or C++.
     New {
@@ -292,14 +298,16 @@ enum Lang {
 }
 
 pub fn lib_main(args: Args) {
-    if let Err(err) = run(args) {
+    if let Err(err) = run_cli(args) {
         eprintln!("\n\n{}", "[ERROR]".bold().red());
         eprintln!("{err:?}");
         std::process::exit(1);
     }
 }
 
-fn run(args: Args) -> eyre::Result<()> {
+fn run_cli(args: Args) -> eyre::Result<()> {
+    tracing_log::LogTracer::init()?;
+
     #[cfg(feature = "tracing")]
     match &args.command {
         Command::Daemon {
@@ -334,7 +342,7 @@ fn run(args: Args) -> eyre::Result<()> {
                 .build()
                 .wrap_err("failed to set up tracing subscriber")?;
         }
-        Command::Run { .. } => {
+        Command::Run { .. } | Command::Build { .. } => {
             let log_level = std::env::var("RUST_LOG").ok().unwrap_or("info".to_string());
             TracingBuilder::new("run")
                 .with_stdout(log_level)
@@ -349,12 +357,6 @@ fn run(args: Args) -> eyre::Result<()> {
         }
     };
 
-    let log_level = env_logger::Builder::new()
-        .filter_level(log::LevelFilter::Info)
-        .parse_default_env()
-        .build()
-        .filter();
-
     match args.command {
         Command::Check {
             dataflow,
@@ -369,9 +371,9 @@ fn run(args: Args) -> eyre::Result<()> {
                     .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
                     .to_owned();
                 Descriptor::blocking_read(&dataflow)?.check(&working_dir)?;
-                check::check_environment((coordinator_addr, coordinator_port).into())?
+                command::check::check_environment((coordinator_addr, coordinator_port).into())?
             }
-            None => check::check_environment((coordinator_addr, coordinator_port).into())?,
+            None => command::check::check_environment((coordinator_addr, coordinator_port).into())?,
         },
         Command::Graph {
             dataflow,
@@ -380,24 +382,20 @@ fn run(args: Args) -> eyre::Result<()> {
         } => {
             graph::create(dataflow, mermaid, open)?;
         }
-        Command::Build { dataflow, uv } => {
-            build::build(dataflow, uv)?;
-        }
+        Command::Build {
+            dataflow,
+            coordinator_addr,
+            coordinator_port,
+            uv,
+            local,
+        } => command::build(dataflow, coordinator_addr, coordinator_port, uv, local)?,
         Command::New {
             args,
             internal_create_with_path_dependencies,
         } => template::create(args, internal_create_with_path_dependencies)?,
-        Command::Run { dataflow, uv } => {
-            let dataflow_path = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-            let rt = Builder::new_multi_thread()
-                .enable_all()
-                .build()
-                .context("tokio runtime failed")?;
-            let result = rt.block_on(Daemon::run_dataflow(&dataflow_path, uv))?;
-            handle_dataflow_result(result, None)?
-        }
+        Command::Run { dataflow, uv } => command::run(dataflow, uv)?,
         Command::Up { config } => {
-            up::up(config.as_deref())?;
+            command::up::up(config.as_deref())?;
         }
         Command::Logs {
             dataflow,
@@ -412,15 +410,16 @@ fn run(args: Args) -> eyre::Result<()> {
             if let Some(dataflow) = dataflow {
                 let uuid = Uuid::parse_str(&dataflow).ok();
                 let name = if uuid.is_some() { None } else { Some(dataflow) };
-                logs::logs(&mut *session, uuid, name, node)?
+                command::logs(&mut *session, uuid, name, node)?
             } else {
-                let active = list.get_active();
+                let active: Vec<dora_message::coordinator_to_cli::DataflowIdAndName> =
+                    list.get_active();
                 let uuid = match &active[..] {
                     [] => bail!("No dataflows are running"),
                     [uuid] => uuid.clone(),
                     _ => inquire::Select::new("Choose dataflow to show logs:", active).prompt()?,
                 };
-                logs::logs(&mut *session, Some(uuid.uuid), None, node)?
+                command::logs(&mut *session, Some(uuid.uuid), None, node)?
             }
         }
         Command::Start {
@@ -433,48 +432,16 @@ fn run(args: Args) -> eyre::Result<()> {
             hot_reload,
             uv,
         } => {
-            let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
-            let dataflow_descriptor =
-                Descriptor::blocking_read(&dataflow).wrap_err("Failed to read yaml dataflow")?;
-            let working_dir = dataflow
-                .canonicalize()
-                .context("failed to canonicalize dataflow path")?
-                .parent()
-                .ok_or_else(|| eyre::eyre!("dataflow path has no parent dir"))?
-                .to_owned();
-
             let coordinator_socket = (coordinator_addr, coordinator_port).into();
-            let mut session = connect_to_coordinator(coordinator_socket)
-                .wrap_err("failed to connect to dora coordinator")?;
-            let dataflow_id = start_dataflow(
-                dataflow_descriptor.clone(),
+            command::start(
+                dataflow,
                 name,
-                working_dir,
-                &mut *session,
+                coordinator_socket,
+                attach,
+                detach,
+                hot_reload,
                 uv,
-            )?;
-
-            let attach = match (attach, detach) {
-                (true, true) => eyre::bail!("both `--attach` and `--detach` are given"),
-                (true, false) => true,
-                (false, true) => false,
-                (false, false) => {
-                    println!("attaching to dataflow (use `--detach` to run in background)");
-                    true
-                }
-            };
-
-            if attach {
-                attach_dataflow(
-                    dataflow_descriptor,
-                    dataflow,
-                    dataflow_id,
-                    &mut *session,
-                    hot_reload,
-                    coordinator_socket,
-                    log_level,
-                )?
-            }
+            )?
         }
         Command::List {
             coordinator_addr,
@@ -504,7 +471,7 @@ fn run(args: Args) -> eyre::Result<()> {
             config,
             coordinator_addr,
             coordinator_port,
-        } => up::destroy(
+        } => command::up::destroy(
             config.as_deref(),
             (coordinator_addr, coordinator_port).into(),
         )?,
@@ -554,8 +521,13 @@ fn run(args: Args) -> eyre::Result<()> {
                                 coordinator_addr
                             );
                         }
+                        let dataflow_session =
+                            DataflowSession::read_session(&dataflow_path).context("failed to read DataflowSession")?;
 
-                        let result = Daemon::run_dataflow(&dataflow_path, false).await?;
+                        let result = Daemon::run_dataflow(&dataflow_path,
+                            dataflow_session.build_id, dataflow_session.local_build, dataflow_session.session_id, false,
+                            LogDestination::Tracing,
+                        ).await?;
                         handle_dataflow_result(result, None)
                     }
                     None => {
@@ -682,37 +654,6 @@ fn run(args: Args) -> eyre::Result<()> {
     Ok(())
 }
 
-fn start_dataflow(
-    dataflow: Descriptor,
-    name: Option<String>,
-    local_working_dir: PathBuf,
-    session: &mut TcpRequestReplyConnection,
-    uv: bool,
-) -> Result<Uuid, eyre::ErrReport> {
-    let reply_raw = session
-        .request(
-            &serde_json::to_vec(&ControlRequest::Start {
-                dataflow,
-                name,
-                local_working_dir,
-                uv,
-            })
-            .unwrap(),
-        )
-        .wrap_err("failed to send start dataflow message")?;
-
-    let result: ControlRequestReply =
-        serde_json::from_slice(&reply_raw).wrap_err("failed to parse reply")?;
-    match result {
-        ControlRequestReply::DataflowStarted { uuid } => {
-            eprintln!("{uuid}");
-            Ok(uuid)
-        }
-        ControlRequestReply::Error(err) => bail!("{err}"),
-        other => bail!("unexpected start dataflow reply: {other:?}"),
-    }
-}
-
 fn stop_dataflow_interactive(
     grace_duration: Option<Duration>,
     session: &mut TcpRequestReplyConnection,
@@ -863,6 +804,8 @@ use pyo3::{
     wrap_pyfunction, Bound, PyResult, Python,
 };
 
+use crate::session::DataflowSession;
+
 #[cfg(feature = "python")]
 #[pyfunction]
 fn py_main(_py: Python) -> PyResult<()> {

binaries/cli/src/output.rs

@@ -0,0 +1,62 @@
+use colored::Colorize;
+use dora_core::build::LogLevelOrStdout;
+use dora_message::common::LogMessage;
+
+pub fn print_log_message(
+    log_message: LogMessage,
+    print_dataflow_id: bool,
+    print_daemon_name: bool,
+) {
+    let LogMessage {
+        build_id: _,
+        dataflow_id,
+        node_id,
+        daemon_id,
+        level,
+        target,
+        module_path: _,
+        file: _,
+        line: _,
+        message,
+    } = log_message;
+    let level = match level {
+        LogLevelOrStdout::LogLevel(level) => match level {
+            log::Level::Error => "ERROR ".red(),
+            log::Level::Warn => "WARN  ".yellow(),
+            log::Level::Info => "INFO  ".green(),
+            log::Level::Debug => "DEBUG ".bright_blue(),
+            log::Level::Trace => "TRACE ".dimmed(),
+        },
+        LogLevelOrStdout::Stdout => "stdout".bright_blue().italic().dimmed(),
+    };
+
+    let dataflow = match dataflow_id {
+        Some(dataflow_id) if print_dataflow_id => format!("dataflow `{dataflow_id}` ").cyan(),
+        _ => String::new().cyan(),
+    };
+    let daemon = match daemon_id {
+        Some(id) if print_daemon_name => match id.machine_id() {
+            Some(machine_id) => format!("on daemon `{machine_id}`"),
+            None => "on default daemon ".to_string(),
+        },
+        None if print_daemon_name => "on default daemon".to_string(),
+        _ => String::new(),
+    }
+    .bright_black();
+    let colon = ":".bright_black().bold();
+    let node = match node_id {
+        Some(node_id) => {
+            let node_id = node_id.to_string().dimmed().bold();
+            let padding = if daemon.is_empty() { "" } else { " " };
+            format!("{node_id}{padding}{daemon}{colon} ")
+        }
+        None if daemon.is_empty() => "".into(),
+        None => format!("{daemon}{colon} "),
+    };
+    let target = match target {
+        Some(target) => format!("{target} ").dimmed(),
+        None => "".normal(),
+    };
+
+    println!("{node}{level} {target}{dataflow}   {message}");
+}

binaries/cli/src/session.rs

@@ -0,0 +1,98 @@
+use std::{
+    collections::BTreeMap,
+    path::{Path, PathBuf},
+};
+
+use dora_core::build::BuildInfo;
+use dora_message::{common::GitSource, id::NodeId, BuildId, SessionId};
+use eyre::{Context, ContextCompat};
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct DataflowSession {
+    pub build_id: Option<BuildId>,
+    pub session_id: SessionId,
+    pub git_sources: BTreeMap<NodeId, GitSource>,
+    pub local_build: Option<BuildInfo>,
+}
+
+impl Default for DataflowSession {
+    fn default() -> Self {
+        Self {
+            build_id: None,
+            session_id: SessionId::generate(),
+            git_sources: Default::default(),
+            local_build: Default::default(),
+        }
+    }
+}
+
+impl DataflowSession {
+    pub fn read_session(dataflow_path: &Path) -> eyre::Result<Self> {
+        let session_file = session_file_path(dataflow_path)?;
+        if session_file.exists() {
+            if let Ok(parsed) = deserialize(&session_file) {
+                return Ok(parsed);
+            } else {
+                tracing::warn!("failed to read dataflow session file, regenerating (you might need to run `dora build` again)");
+            }
+        }
+
+        let default_session = DataflowSession::default();
+        default_session.write_out_for_dataflow(dataflow_path)?;
+        Ok(default_session)
+    }
+
+    pub fn write_out_for_dataflow(&self, dataflow_path: &Path) -> eyre::Result<()> {
+        let session_file = session_file_path(dataflow_path)?;
+        let filename = session_file
+            .file_name()
+            .context("session file has no file name")?
+            .to_str()
+            .context("session file name is no utf8")?;
+        if let Some(parent) = session_file.parent() {
+            std::fs::create_dir_all(parent).context("failed to create out dir")?;
+        }
+        std::fs::write(&session_file, self.serialize()?)
+            .context("failed to write dataflow session file")?;
+        let gitignore = session_file.with_file_name(".gitignore");
+        if gitignore.exists() {
+            let existing =
+                std::fs::read_to_string(&gitignore).context("failed to read gitignore")?;
+            if !existing
+                .lines()
+                .any(|l| l.split_once('/') == Some(("", filename)))
+            {
+                let new = existing + &format!("\n/{filename}\n");
+                std::fs::write(gitignore, new).context("failed to update gitignore")?;
+            }
+        } else {
+            std::fs::write(gitignore, format!("/{filename}\n"))
+                .context("failed to write gitignore")?;
+        }
+        Ok(())
+    }
+
+    fn serialize(&self) -> eyre::Result<String> {
+        serde_yaml::to_string(&self).context("failed to serialize dataflow session file")
+    }
+}
+
+fn deserialize(session_file: &Path) -> eyre::Result<DataflowSession> {
+    std::fs::read_to_string(session_file)
+        .context("failed to read DataflowSession file")
+        .and_then(|s| {
+            serde_yaml::from_str(&s).context("failed to deserialize DataflowSession file")
+        })
+}
+
+fn session_file_path(dataflow_path: &Path) -> eyre::Result<PathBuf> {
+    let file_stem = dataflow_path
+        .file_stem()
+        .wrap_err("dataflow path has no file stem")?
+        .to_str()
+        .wrap_err("dataflow file stem is not valid utf-8")?;
+    let session_file = dataflow_path
+        .with_file_name("out")
+        .join(format!("{file_stem}.dora-session.yaml"));
+    Ok(session_file)
+}

binaries/cli/src/template/c/cmake-template.txt

@@ -64,16 +64,16 @@ link_directories(${dora_link_dirs})
 add_executable(talker_1 talker_1/node.c)
 add_dependencies(talker_1 Dora_c)
 target_include_directories(talker_1 PRIVATE ${dora_c_include_dir})
-target_link_libraries(talker_1 dora_node_api_c m)
+target_link_libraries(talker_1 dora_node_api_c m z)
 
 add_executable(talker_2 talker_2/node.c)
 add_dependencies(talker_2 Dora_c)
 target_include_directories(talker_2 PRIVATE ${dora_c_include_dir})
-target_link_libraries(talker_2 dora_node_api_c m)
+target_link_libraries(talker_2 dora_node_api_c m z)
 
 add_executable(listener_1 listener_1/node.c)
 add_dependencies(listener_1 Dora_c)
 target_include_directories(listener_1 PRIVATE ${dora_c_include_dir})
-target_link_libraries(listener_1 dora_node_api_c m)
+target_link_libraries(listener_1 dora_node_api_c m z)
 
-install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)

binaries/cli/src/template/cxx/cmake-template.txt

@@ -70,16 +70,16 @@ link_directories(${dora_link_dirs})
 add_executable(talker_1 talker_1/node.cc ${node_bridge})
 add_dependencies(talker_1 Dora_cxx)
 target_include_directories(talker_1 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(talker_1 dora_node_api_cxx)
+target_link_libraries(talker_1 dora_node_api_cxx z)
 
 add_executable(talker_2 talker_2/node.cc ${node_bridge})
 add_dependencies(talker_2 Dora_cxx)
 target_include_directories(talker_2 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(talker_2 dora_node_api_cxx)
+target_link_libraries(talker_2 dora_node_api_cxx z)
 
 add_executable(listener_1 listener_1/node.cc ${node_bridge})
 add_dependencies(listener_1 Dora_cxx)
 target_include_directories(listener_1 PRIVATE ${dora_cxx_include_dir})
-target_link_libraries(listener_1 dora_node_api_cxx)
+target_link_libraries(listener_1 dora_node_api_cxx z)
 
-install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+install(TARGETS listener_1 talker_1 talker_2 DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)

binaries/coordinator/src/control.rs

@@ -2,7 +2,9 @@ use crate::{
     tcp_utils::{tcp_receive, tcp_send},
     Event,
 };
-use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
+use dora_message::{
+    cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply, BuildId,
+};
 use eyre::{eyre, Context};
 use futures::{
     future::{self, Either},
@@ -79,6 +81,7 @@ async fn handle_requests(
     tx: mpsc::Sender<ControlEvent>,
     _finish_tx: mpsc::Sender<()>,
 ) {
+    let peer_addr = connection.peer_addr().ok();
     loop {
         let next_request = tcp_receive(&mut connection).map(Either::Left);
         let coordinator_stopped = tx.closed().map(Either::Right);
@@ -114,11 +117,29 @@ async fn handle_requests(
             break;
         }
 
-        let result = match request {
+        if let Ok(ControlRequest::BuildLogSubscribe { build_id, level }) = request {
+            let _ = tx
+                .send(ControlEvent::BuildLogSubscribe {
+                    build_id,
+                    level,
+                    connection,
+                })
+                .await;
+            break;
+        }
+
+        let mut result = match request {
             Ok(request) => handle_request(request, &tx).await,
             Err(err) => Err(err),
         };
 
+        if let Ok(ControlRequestReply::CliAndDefaultDaemonIps { cli, .. }) = &mut result {
+            if cli.is_none() {
+                // fill cli IP address in reply
+                *cli = peer_addr.map(|s| s.ip());
+            }
+        }
+
         let reply = result.unwrap_or_else(|err| ControlRequestReply::Error(format!("{err:?}")));
         let serialized: Vec<u8> =
             match serde_json::to_vec(&reply).wrap_err("failed to serialize ControlRequestReply") {
@@ -155,7 +176,7 @@ async fn handle_request(
 ) -> eyre::Result<ControlRequestReply> {
     let (reply_tx, reply_rx) = oneshot::channel();
     let event = ControlEvent::IncomingRequest {
-        request,
+        request: request.clone(),
         reply_sender: reply_tx,
     };
 
@@ -165,7 +186,7 @@ async fn handle_request(
 
     reply_rx
         .await
-        .unwrap_or(Ok(ControlRequestReply::CoordinatorStopped))
+        .wrap_err_with(|| format!("no coordinator reply to {request:?}"))?
 }
 
 #[derive(Debug)]
@@ -179,6 +200,11 @@ pub enum ControlEvent {
         level: log::LevelFilter,
         connection: TcpStream,
     },
+    BuildLogSubscribe {
+        build_id: BuildId,
+        level: log::LevelFilter,
+        connection: TcpStream,
+    },
     Error(eyre::Report),
 }
 

binaries/coordinator/src/lib.rs

@@ -5,22 +5,27 @@ use crate::{
 pub use control::ControlEvent;
 use dora_core::{
     config::{NodeId, OperatorId},
+    descriptor::DescriptorExt,
     uhlc::{self, HLC},
 };
 use dora_message::{
     cli_to_coordinator::ControlRequest,
-    common::DaemonId,
+    common::{DaemonId, GitSource},
     coordinator_to_cli::{
         ControlRequestReply, DataflowIdAndName, DataflowList, DataflowListEntry, DataflowResult,
         DataflowStatus, LogLevel, LogMessage,
     },
-    coordinator_to_daemon::{DaemonCoordinatorEvent, RegisterResult, Timestamped},
+    coordinator_to_daemon::{
+        BuildDataflowNodes, DaemonCoordinatorEvent, RegisterResult, Timestamped,
+    },
     daemon_to_coordinator::{DaemonCoordinatorReply, DataflowDaemonResult},
     descriptor::{Descriptor, ResolvedNode},
+    BuildId, DataflowId, SessionId,
 };
 use eyre::{bail, eyre, ContextCompat, Result, WrapErr};
 use futures::{future::join_all, stream::FuturesUnordered, Future, Stream, StreamExt};
 use futures_concurrency::stream::Merge;
+use itertools::Itertools;
 use log_subscriber::LogSubscriber;
 use run::SpawnedDataflow;
 use std::{
@@ -30,7 +35,11 @@ use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
-use tokio::{net::TcpStream, sync::mpsc, task::JoinHandle};
+use tokio::{
+    net::TcpStream,
+    sync::{mpsc, oneshot},
+    task::JoinHandle,
+};
 use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};
 use uuid::Uuid;
 
@@ -135,6 +144,10 @@ impl DaemonConnections {
         }
     }
 
+    fn get(&self, id: &DaemonId) -> Option<&DaemonConnection> {
+        self.daemons.get(id)
+    }
+
     fn get_mut(&mut self, id: &DaemonId) -> Option<&mut DaemonConnection> {
         self.daemons.get_mut(id)
     }
@@ -157,10 +170,6 @@ impl DaemonConnections {
         self.daemons.keys()
     }
 
-    fn iter(&self) -> impl Iterator<Item = (&DaemonId, &DaemonConnection)> {
-        self.daemons.iter()
-    }
-
     fn iter_mut(&mut self) -> impl Iterator<Item = (&DaemonId, &mut DaemonConnection)> {
         self.daemons.iter_mut()
     }
@@ -194,13 +203,20 @@ async fn start_inner(
 
     let mut events = (abortable_events, daemon_events).merge();
 
-    let mut running_dataflows: HashMap<Uuid, RunningDataflow> = HashMap::new();
-    let mut dataflow_results: HashMap<Uuid, BTreeMap<DaemonId, DataflowDaemonResult>> =
+    let mut running_builds: HashMap<BuildId, RunningBuild> = HashMap::new();
+    let mut finished_builds: HashMap<BuildId, CachedResult> = HashMap::new();
+
+    let mut running_dataflows: HashMap<DataflowId, RunningDataflow> = HashMap::new();
+    let mut dataflow_results: HashMap<DataflowId, BTreeMap<DaemonId, DataflowDaemonResult>> =
         HashMap::new();
-    let mut archived_dataflows: HashMap<Uuid, ArchivedDataflow> = HashMap::new();
+    let mut archived_dataflows: HashMap<DataflowId, ArchivedDataflow> = HashMap::new();
     let mut daemon_connections = DaemonConnections::default();
 
     while let Some(event) = events.next().await {
+        // used below for measuring the event handling duration
+        let start = Instant::now();
+        let event_kind = event.kind();
+
         if event.log() {
             tracing::trace!("Handling event {event:?}");
         }
@@ -347,12 +363,13 @@ async fn start_inner(
                                 let mut finished_dataflow = entry.remove();
                                 let dataflow_id = finished_dataflow.uuid;
                                 send_log_message(
-                                    &mut finished_dataflow,
+                                    &mut finished_dataflow.log_subscribers,
                                     &LogMessage {
-                                        dataflow_id,
+                                        build_id: None,
+                                        dataflow_id: Some(dataflow_id),
                                         node_id: None,
                                         daemon_id: None,
-                                        level: LogLevel::Info,
+                                        level: LogLevel::Info.into(),
                                         target: Some("coordinator".into()),
                                         module_path: None,
                                         file: None,
@@ -371,9 +388,15 @@ async fn start_inner(
                                             DataflowResult::ok_empty(uuid, clock.new_timestamp())
                                         }),
                                 };
-                                for sender in finished_dataflow.reply_senders {
+                                for sender in finished_dataflow.stop_reply_senders {
                                     let _ = sender.send(Ok(reply.clone()));
                                 }
+                                if !matches!(
+                                    finished_dataflow.spawn_result,
+                                    CachedResult::Cached { .. }
+                                ) {
+                                    log::error!("pending spawn result on dataflow finish");
+                                }
                             }
                         }
                         std::collections::hash_map::Entry::Vacant(_) => {
@@ -389,7 +412,54 @@ async fn start_inner(
                     reply_sender,
                 } => {
                     match request {
+                        ControlRequest::Build {
+                            session_id,
+                            dataflow,
+                            git_sources,
+                            prev_git_sources,
+                            local_working_dir,
+                            uv,
+                        } => {
+                            // assign a random build id
+                            let build_id = BuildId::generate();
+
+                            let result = build_dataflow(
+                                build_id,
+                                session_id,
+                                dataflow,
+                                git_sources,
+                                prev_git_sources,
+                                local_working_dir,
+                                &clock,
+                                uv,
+                                &mut daemon_connections,
+                            )
+                            .await;
+                            match result {
+                                Ok(build) => {
+                                    running_builds.insert(build_id, build);
+                                    let _ = reply_sender.send(Ok(
+                                        ControlRequestReply::DataflowBuildTriggered { build_id },
+                                    ));
+                                }
+                                Err(err) => {
+                                    let _ = reply_sender.send(Err(err));
+                                }
+                            }
+                        }
+                        ControlRequest::WaitForBuild { build_id } => {
+                            if let Some(build) = running_builds.get_mut(&build_id) {
+                                build.build_result.register(reply_sender);
+                            } else if let Some(result) = finished_builds.get_mut(&build_id) {
+                                result.register(reply_sender);
+                            } else {
+                                let _ =
+                                    reply_sender.send(Err(eyre!("unknown build id {build_id}")));
+                            }
+                        }
                         ControlRequest::Start {
+                            build_id,
+                            session_id,
                             dataflow,
                             name,
                             local_working_dir,
@@ -408,6 +478,8 @@ async fn start_inner(
                                     }
                                 }
                                 let dataflow = start_dataflow(
+                                    build_id,
+                                    session_id,
                                     dataflow,
                                     local_working_dir,
                                     name,
@@ -418,16 +490,30 @@ async fn start_inner(
                                 .await?;
                                 Ok(dataflow)
                             };
-                            let reply = inner.await.map(|dataflow| {
-                                let uuid = dataflow.uuid;
-                                running_dataflows.insert(uuid, dataflow);
-                                ControlRequestReply::DataflowStarted { uuid }
-                            });
-                            let _ = reply_sender.send(reply);
+                            match inner.await {
+                                Ok(dataflow) => {
+                                    let uuid = dataflow.uuid;
+                                    running_dataflows.insert(uuid, dataflow);
+                                    let _ = reply_sender.send(Ok(
+                                        ControlRequestReply::DataflowStartTriggered { uuid },
+                                    ));
+                                }
+                                Err(err) => {
+                                    let _ = reply_sender.send(Err(err));
+                                }
+                            }
+                        }
+                        ControlRequest::WaitForSpawn { dataflow_id } => {
+                            if let Some(dataflow) = running_dataflows.get_mut(&dataflow_id) {
+                                dataflow.spawn_result.register(reply_sender);
+                            } else {
+                                let _ =
+                                    reply_sender.send(Err(eyre!("unknown dataflow {dataflow_id}")));
+                            }
                         }
                         ControlRequest::Check { dataflow_uuid } => {
                             let status = match &running_dataflows.get(&dataflow_uuid) {
-                                Some(_) => ControlRequestReply::DataflowStarted {
+                                Some(_) => ControlRequestReply::DataflowSpawned {
                                     uuid: dataflow_uuid,
                                 },
                                 None => ControlRequestReply::DataflowStopped {
@@ -495,7 +581,7 @@ async fn start_inner(
 
                             match dataflow {
                                 Ok(dataflow) => {
-                                    dataflow.reply_senders.push(reply_sender);
+                                    dataflow.stop_reply_senders.push(reply_sender);
                                 }
                                 Err(err) => {
                                     let _ = reply_sender.send(Err(err));
@@ -528,7 +614,7 @@ async fn start_inner(
 
                                 match dataflow {
                                     Ok(dataflow) => {
-                                        dataflow.reply_senders.push(reply_sender);
+                                        dataflow.stop_reply_senders.push(reply_sender);
                                     }
                                     Err(err) => {
                                         let _ = reply_sender.send(Err(err));
@@ -626,6 +712,27 @@ async fn start_inner(
                                 "LogSubscribe request should be handled separately"
                             )));
                         }
+                        ControlRequest::BuildLogSubscribe { .. } => {
+                            let _ = reply_sender.send(Err(eyre::eyre!(
+                                "BuildLogSubscribe request should be handled separately"
+                            )));
+                        }
+                        ControlRequest::CliAndDefaultDaemonOnSameMachine => {
+                            let mut default_daemon_ip = None;
+                            if let Some(default_id) = daemon_connections.unnamed().next() {
+                                if let Some(connection) = daemon_connections.get(default_id) {
+                                    if let Ok(addr) = connection.stream.peer_addr() {
+                                        default_daemon_ip = Some(addr.ip());
+                                    }
+                                }
+                            }
+                            let _ = reply_sender.send(Ok(
+                                ControlRequestReply::CliAndDefaultDaemonIps {
+                                    default_daemon: default_daemon_ip,
+                                    cli: None, // filled later
+                                },
+                            ));
+                        }
                     }
                 }
                 ControlEvent::Error(err) => tracing::error!("{err:?}"),
@@ -640,6 +747,17 @@ async fn start_inner(
                             .push(LogSubscriber::new(level, connection));
                     }
                 }
+                ControlEvent::BuildLogSubscribe {
+                    build_id,
+                    level,
+                    connection,
+                } => {
+                    if let Some(build) = running_builds.get_mut(&build_id) {
+                        build
+                            .log_subscribers
+                            .push(LogSubscriber::new(level, connection));
+                    }
+                }
             },
             Event::DaemonHeartbeatInterval => {
                 let mut disconnected = BTreeSet::new();
@@ -695,14 +813,89 @@ async fn start_inner(
                 }
             }
             Event::Log(message) => {
-                if let Some(dataflow) = running_dataflows.get_mut(&message.dataflow_id) {
-                    send_log_message(dataflow, &message).await;
+                if let Some(dataflow_id) = &message.dataflow_id {
+                    if let Some(dataflow) = running_dataflows.get_mut(dataflow_id) {
+                        send_log_message(&mut dataflow.log_subscribers, &message).await;
+                    }
+                }
+                if let Some(build_id) = message.build_id {
+                    if let Some(build) = running_builds.get_mut(&build_id) {
+                        send_log_message(&mut build.log_subscribers, &message).await;
+                    }
                 }
             }
             Event::DaemonExit { daemon_id } => {
                 tracing::info!("Daemon `{daemon_id}` exited");
                 daemon_connections.remove(&daemon_id);
             }
+            Event::DataflowBuildResult {
+                build_id,
+                daemon_id,
+                result,
+            } => match running_builds.get_mut(&build_id) {
+                Some(build) => {
+                    build.pending_build_results.remove(&daemon_id);
+                    match result {
+                        Ok(()) => {}
+                        Err(err) => {
+                            build.errors.push(format!("{err:?}"));
+                        }
+                    };
+                    if build.pending_build_results.is_empty() {
+                        tracing::info!("dataflow build finished: `{build_id}`");
+                        let mut build = running_builds.remove(&build_id).unwrap();
+                        let result = if build.errors.is_empty() {
+                            Ok(())
+                        } else {
+                            Err(format!("build failed: {}", build.errors.join("\n\n")))
+                        };
+
+                        build.build_result.set_result(Ok(
+                            ControlRequestReply::DataflowBuildFinished { build_id, result },
+                        ));
+
+                        finished_builds.insert(build_id, build.build_result);
+                    }
+                }
+                None => {
+                    tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
+                }
+            },
+            Event::DataflowSpawnResult {
+                dataflow_id,
+                daemon_id,
+                result,
+            } => match running_dataflows.get_mut(&dataflow_id) {
+                Some(dataflow) => {
+                    dataflow.pending_spawn_results.remove(&daemon_id);
+                    match result {
+                        Ok(()) => {
+                            if dataflow.pending_spawn_results.is_empty() {
+                                tracing::info!("successfully spawned dataflow `{dataflow_id}`",);
+                                dataflow.spawn_result.set_result(Ok(
+                                    ControlRequestReply::DataflowSpawned { uuid: dataflow_id },
+                                ));
+                            }
+                        }
+                        Err(err) => {
+                            tracing::warn!("error while spawning dataflow `{dataflow_id}`");
+                            dataflow.spawn_result.set_result(Err(err));
+                        }
+                    };
+                }
+                None => {
+                    tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
+                }
+            },
+        }
+
+        // warn if event handling took too long -> the main loop should never be blocked for too long
+        let elapsed = start.elapsed();
+        if elapsed > Duration::from_millis(100) {
+            tracing::warn!(
+                "Coordinator took {}ms for handling event: {event_kind}",
+                elapsed.as_millis()
+            );
         }
     }
 
@@ -711,8 +904,8 @@ async fn start_inner(
     Ok(())
 }
 
-async fn send_log_message(dataflow: &mut RunningDataflow, message: &LogMessage) {
-    for subscriber in &mut dataflow.log_subscribers {
+async fn send_log_message(log_subscribers: &mut Vec<LogSubscriber>, message: &LogMessage) {
+    for subscriber in log_subscribers.iter_mut() {
         let send_result =
             tokio::time::timeout(Duration::from_millis(100), subscriber.send_message(message));
 
@@ -720,7 +913,7 @@ async fn send_log_message(dataflow: &mut RunningDataflow, message: &LogMessage)
             subscriber.close();
         }
     }
-    dataflow.log_subscribers.retain(|s| !s.is_closed());
+    log_subscribers.retain(|s| !s.is_closed());
 }
 
 fn dataflow_result(
@@ -787,6 +980,15 @@ async fn send_heartbeat_message(
         .wrap_err("failed to send heartbeat message to daemon")
 }
 
+struct RunningBuild {
+    errors: Vec<String>,
+    build_result: CachedResult,
+
+    log_subscribers: Vec<LogSubscriber>,
+
+    pending_build_results: BTreeSet<DaemonId>,
+}
+
 struct RunningDataflow {
     name: Option<String>,
     uuid: Uuid,
@@ -797,9 +999,66 @@ struct RunningDataflow {
     exited_before_subscribe: Vec<NodeId>,
     nodes: BTreeMap<NodeId, ResolvedNode>,
 
-    reply_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
+    spawn_result: CachedResult,
+    stop_reply_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
 
     log_subscribers: Vec<LogSubscriber>,
+
+    pending_spawn_results: BTreeSet<DaemonId>,
+}
+
+pub enum CachedResult {
+    Pending {
+        result_senders: Vec<tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>>,
+    },
+    Cached {
+        result: eyre::Result<ControlRequestReply>,
+    },
+}
+
+impl Default for CachedResult {
+    fn default() -> Self {
+        Self::Pending {
+            result_senders: Vec::new(),
+        }
+    }
+}
+
+impl CachedResult {
+    fn register(
+        &mut self,
+        reply_sender: tokio::sync::oneshot::Sender<eyre::Result<ControlRequestReply>>,
+    ) {
+        match self {
+            CachedResult::Pending { result_senders } => result_senders.push(reply_sender),
+            CachedResult::Cached { result } => {
+                Self::send_result_to(result, reply_sender);
+            }
+        }
+    }
+
+    fn set_result(&mut self, result: eyre::Result<ControlRequestReply>) {
+        match self {
+            CachedResult::Pending { result_senders } => {
+                for sender in result_senders.drain(..) {
+                    Self::send_result_to(&result, sender);
+                }
+                *self = CachedResult::Cached { result };
+            }
+            CachedResult::Cached { .. } => {}
+        }
+    }
+
+    fn send_result_to(
+        result: &eyre::Result<ControlRequestReply>,
+        sender: oneshot::Sender<eyre::Result<ControlRequestReply>>,
+    ) {
+        let result = match result {
+            Ok(r) => Ok(r.clone()),
+            Err(err) => Err(eyre!("{err:?}")),
+        };
+        let _ = sender.send(result);
+    }
 }
 
 struct ArchivedDataflow {
@@ -943,7 +1202,7 @@ async fn retrieve_logs(
     let machine_ids: Vec<Option<String>> = nodes
         .values()
         .filter(|node| node.id == node_id)
-        .map(|node| node.deploy.machine.clone())
+        .map(|node| node.deploy.as_ref().and_then(|d| d.machine.clone()))
         .collect();
 
     let machine_id = if let [machine_id] = &machine_ids[..] {
@@ -992,9 +1251,127 @@ async fn retrieve_logs(
     reply_logs.map_err(|err| eyre!(err))
 }
 
+#[allow(clippy::too_many_arguments)]
+#[tracing::instrument(skip(daemon_connections, clock))]
+async fn build_dataflow(
+    build_id: BuildId,
+    session_id: SessionId,
+    dataflow: Descriptor,
+    git_sources: BTreeMap<NodeId, GitSource>,
+    prev_git_sources: BTreeMap<NodeId, GitSource>,
+    local_working_dir: Option<PathBuf>,
+    clock: &HLC,
+    uv: bool,
+    daemon_connections: &mut DaemonConnections,
+) -> eyre::Result<RunningBuild> {
+    let nodes = dataflow.resolve_aliases_and_set_defaults()?;
+
+    let mut git_sources_by_daemon = git_sources
+        .into_iter()
+        .into_grouping_map_by(|(id, _)| {
+            nodes
+                .get(id)
+                .and_then(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()))
+        })
+        .collect();
+    let mut prev_git_sources_by_daemon = prev_git_sources
+        .into_iter()
+        .into_grouping_map_by(|(id, _)| {
+            nodes
+                .get(id)
+                .and_then(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()))
+        })
+        .collect();
+
+    let nodes_by_daemon = nodes
+        .values()
+        .into_group_map_by(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()));
+
+    let mut daemons = BTreeSet::new();
+    for (machine, nodes_on_machine) in &nodes_by_daemon {
+        let nodes_on_machine = nodes_on_machine.iter().map(|n| n.id.clone()).collect();
+        tracing::debug!(
+            "Running dataflow build `{build_id}` on machine `{machine:?}` (nodes: {nodes_on_machine:?})"
+        );
+
+        let build_command = BuildDataflowNodes {
+            build_id,
+            session_id,
+            local_working_dir: local_working_dir.clone(),
+            git_sources: git_sources_by_daemon.remove(machine).unwrap_or_default(),
+            prev_git_sources: prev_git_sources_by_daemon
+                .remove(machine)
+                .unwrap_or_default(),
+            dataflow_descriptor: dataflow.clone(),
+            nodes_on_machine,
+            uv,
+        };
+        let message = serde_json::to_vec(&Timestamped {
+            inner: DaemonCoordinatorEvent::Build(build_command),
+            timestamp: clock.new_timestamp(),
+        })?;
+
+        let daemon_id =
+            build_dataflow_on_machine(daemon_connections, machine.map(|s| s.as_str()), &message)
+                .await
+                .wrap_err_with(|| format!("failed to build dataflow on machine `{machine:?}`"))?;
+        daemons.insert(daemon_id);
+    }
+
+    tracing::info!("successfully triggered dataflow build `{build_id}`",);
+
+    Ok(RunningBuild {
+        errors: Vec::new(),
+        build_result: CachedResult::default(),
+        log_subscribers: Vec::new(),
+        pending_build_results: daemons,
+    })
+}
+
+async fn build_dataflow_on_machine(
+    daemon_connections: &mut DaemonConnections,
+    machine: Option<&str>,
+    message: &[u8],
+) -> Result<DaemonId, eyre::ErrReport> {
+    let daemon_id = match machine {
+        Some(machine) => daemon_connections
+            .get_matching_daemon_id(machine)
+            .wrap_err_with(|| format!("no matching daemon for machine id {machine:?}"))?
+            .clone(),
+        None => daemon_connections
+            .unnamed()
+            .next()
+            .wrap_err("no unnamed daemon connections")?
+            .clone(),
+    };
+
+    let daemon_connection = daemon_connections
+        .get_mut(&daemon_id)
+        .wrap_err_with(|| format!("no daemon connection for daemon `{daemon_id}`"))?;
+    tcp_send(&mut daemon_connection.stream, message)
+        .await
+        .wrap_err("failed to send build message to daemon")?;
+
+    let reply_raw = tcp_receive(&mut daemon_connection.stream)
+        .await
+        .wrap_err("failed to receive build reply from daemon")?;
+    match serde_json::from_slice(&reply_raw)
+        .wrap_err("failed to deserialize build reply from daemon")?
+    {
+        DaemonCoordinatorReply::TriggerBuildResult(result) => result
+            .map_err(|e| eyre!(e))
+            .wrap_err("daemon returned an error")?,
+        _ => bail!("unexpected reply"),
+    }
+    Ok(daemon_id)
+}
+
+#[allow(clippy::too_many_arguments)]
 async fn start_dataflow(
+    build_id: Option<BuildId>,
+    session_id: SessionId,
     dataflow: Descriptor,
-    working_dir: PathBuf,
+    local_working_dir: Option<PathBuf>,
     name: Option<String>,
     daemon_connections: &mut DaemonConnections,
     clock: &HLC,
@@ -1004,7 +1381,16 @@ async fn start_dataflow(
         uuid,
         daemons,
         nodes,
-    } = spawn_dataflow(dataflow, working_dir, daemon_connections, clock, uv).await?;
+    } = spawn_dataflow(
+        build_id,
+        session_id,
+        dataflow,
+        local_working_dir,
+        daemon_connections,
+        clock,
+        uv,
+    )
+    .await?;
     Ok(RunningDataflow {
         uuid,
         name,
@@ -1014,10 +1400,12 @@ async fn start_dataflow(
             BTreeSet::new()
         },
         exited_before_subscribe: Default::default(),
-        daemons,
+        daemons: daemons.clone(),
         nodes,
-        reply_senders: Vec::new(),
+        spawn_result: CachedResult::default(),
+        stop_reply_senders: Vec::new(),
         log_subscribers: Vec::new(),
+        pending_spawn_results: daemons,
     })
 }
 
@@ -1092,6 +1480,16 @@ pub enum Event {
     DaemonExit {
         daemon_id: dora_message::common::DaemonId,
     },
+    DataflowBuildResult {
+        build_id: BuildId,
+        daemon_id: DaemonId,
+        result: eyre::Result<()>,
+    },
+    DataflowSpawnResult {
+        dataflow_id: uuid::Uuid,
+        daemon_id: DaemonId,
+        result: eyre::Result<()>,
+    },
 }
 
 impl Event {
@@ -1103,6 +1501,23 @@ impl Event {
             _ => true,
         }
     }
+
+    fn kind(&self) -> &'static str {
+        match self {
+            Event::NewDaemonConnection(_) => "NewDaemonConnection",
+            Event::DaemonConnectError(_) => "DaemonConnectError",
+            Event::DaemonHeartbeat { .. } => "DaemonHeartbeat",
+            Event::Dataflow { .. } => "Dataflow",
+            Event::Control(_) => "Control",
+            Event::Daemon(_) => "Daemon",
+            Event::DaemonHeartbeatInterval => "DaemonHeartbeatInterval",
+            Event::CtrlC => "CtrlC",
+            Event::Log(_) => "Log",
+            Event::DaemonExit { .. } => "DaemonExit",
+            Event::DataflowBuildResult { .. } => "DataflowBuildResult",
+            Event::DataflowSpawnResult { .. } => "DataflowSpawnResult",
+        }
+    }
 }
 
 #[derive(Debug)]

binaries/coordinator/src/listener.rs

@@ -112,6 +112,29 @@ pub async fn handle_connection(
                         break;
                     }
                 }
+                DaemonEvent::BuildResult { build_id, result } => {
+                    let event = Event::DataflowBuildResult {
+                        build_id,
+                        daemon_id,
+                        result: result.map_err(|err| eyre::eyre!(err)),
+                    };
+                    if events_tx.send(event).await.is_err() {
+                        break;
+                    }
+                }
+                DaemonEvent::SpawnResult {
+                    dataflow_id,
+                    result,
+                } => {
+                    let event = Event::DataflowSpawnResult {
+                        dataflow_id,
+                        daemon_id,
+                        result: result.map_err(|err| eyre::eyre!(err)),
+                    };
+                    if events_tx.send(event).await.is_err() {
+                        break;
+                    }
+                }
             },
         };
     }

binaries/coordinator/src/log_subscriber.rs

@@ -17,9 +17,15 @@ impl LogSubscriber {
     }
 
     pub async fn send_message(&mut self, message: &LogMessage) -> eyre::Result<()> {
-        if message.level > self.level {
-            return Ok(());
+        match message.level {
+            dora_core::build::LogLevelOrStdout::LogLevel(level) => {
+                if level > self.level {
+                    return Ok(());
+                }
+            }
+            dora_core::build::LogLevelOrStdout::Stdout => {}
         }
+
         let message = serde_json::to_vec(&message)?;
         let connection = self.connection.as_mut().context("connection is closed")?;
         tcp_send(connection, &message)

binaries/coordinator/src/run/mod.rs

@@ -10,6 +10,7 @@ use dora_message::{
     daemon_to_coordinator::DaemonCoordinatorReply,
     descriptor::{Descriptor, ResolvedNode},
     id::NodeId,
+    BuildId, SessionId,
 };
 use eyre::{bail, eyre, ContextCompat, WrapErr};
 use itertools::Itertools;
@@ -21,8 +22,10 @@ use uuid::{NoContext, Timestamp, Uuid};
 
 #[tracing::instrument(skip(daemon_connections, clock))]
 pub(super) async fn spawn_dataflow(
+    build_id: Option<BuildId>,
+    session_id: SessionId,
     dataflow: Descriptor,
-    working_dir: PathBuf,
+    local_working_dir: Option<PathBuf>,
     daemon_connections: &mut DaemonConnections,
     clock: &HLC,
     uv: bool,
@@ -30,7 +33,9 @@ pub(super) async fn spawn_dataflow(
     let nodes = dataflow.resolve_aliases_and_set_defaults()?;
     let uuid = Uuid::new_v7(Timestamp::now(NoContext));
 
-    let nodes_by_daemon = nodes.values().into_group_map_by(|n| &n.deploy.machine);
+    let nodes_by_daemon = nodes
+        .values()
+        .into_group_map_by(|n| n.deploy.as_ref().and_then(|d| d.machine.as_ref()));
 
     let mut daemons = BTreeSet::new();
     for (machine, nodes_on_machine) in &nodes_by_daemon {
@@ -40,8 +45,10 @@ pub(super) async fn spawn_dataflow(
         );
 
         let spawn_command = SpawnDataflowNodes {
+            build_id,
+            session_id,
             dataflow_id: uuid,
-            working_dir: working_dir.clone(),
+            local_working_dir: local_working_dir.clone(),
             nodes: nodes.clone(),
             dataflow_descriptor: dataflow.clone(),
             spawn_nodes,
@@ -52,13 +59,14 @@ pub(super) async fn spawn_dataflow(
             timestamp: clock.new_timestamp(),
         })?;
 
-        let daemon_id = spawn_dataflow_on_machine(daemon_connections, machine.as_deref(), &message)
-            .await
-            .wrap_err_with(|| format!("failed to spawn dataflow on machine `{machine:?}`"))?;
+        let daemon_id =
+            spawn_dataflow_on_machine(daemon_connections, machine.map(|m| m.as_str()), &message)
+                .await
+                .wrap_err_with(|| format!("failed to spawn dataflow on machine `{machine:?}`"))?;
         daemons.insert(daemon_id);
     }
 
-    tracing::info!("successfully spawned dataflow `{uuid}`");
+    tracing::info!("successfully triggered dataflow spawn `{uuid}`",);
 
     Ok(SpawnedDataflow {
         uuid,
@@ -90,13 +98,14 @@ async fn spawn_dataflow_on_machine(
     tcp_send(&mut daemon_connection.stream, message)
         .await
         .wrap_err("failed to send spawn message to daemon")?;
+
     let reply_raw = tcp_receive(&mut daemon_connection.stream)
         .await
         .wrap_err("failed to receive spawn reply from daemon")?;
     match serde_json::from_slice(&reply_raw)
         .wrap_err("failed to deserialize spawn reply from daemon")?
     {
-        DaemonCoordinatorReply::SpawnResult(result) => result
+        DaemonCoordinatorReply::TriggerSpawnResult(result) => result
             .map_err(|e| eyre!(e))
             .wrap_err("daemon returned an error")?,
         _ => bail!("unexpected reply"),

binaries/daemon/Cargo.toml

@@ -24,14 +24,14 @@ tracing = "0.1.36"
 tracing-opentelemetry = { version = "0.18.0", optional = true }
 futures-concurrency = "7.1.0"
 serde_json = "1.0.86"
-dora-core = { workspace = true }
+dora-core = { workspace = true, features = ["build"] }
 flume = "0.10.14"
 dora-download = { workspace = true }
 dora-tracing = { workspace = true, optional = true }
 dora-arrow-convert = { workspace = true }
 dora-node-api = { workspace = true }
 dora-message = { workspace = true }
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 uuid = { version = "1.7", features = ["v7"] }
 futures = "0.3.25"
 shared-memory-server = { workspace = true }
@@ -44,3 +44,7 @@ sysinfo = "0.30.11"
 crossbeam = "0.8.4"
 crossbeam-skiplist = "0.1.3"
 zenoh = "1.1.1"
+url = "2.5.4"
+git2 = { workspace = true }
+dunce = "1.0.5"
+itertools = "0.14"

binaries/daemon/src/log.rs

@@ -1,14 +1,21 @@
 use std::{
+    ops::{Deref, DerefMut},
     path::{Path, PathBuf},
     sync::Arc,
 };
 
-use dora_core::{config::NodeId, uhlc};
+use dora_core::{
+    build::{BuildLogger, LogLevelOrStdout},
+    config::NodeId,
+    uhlc,
+};
 use dora_message::{
     common::{DaemonId, LogLevel, LogMessage, Timestamped},
     daemon_to_coordinator::{CoordinatorRequest, DaemonEvent},
+    BuildId,
 };
 use eyre::Context;
+use flume::Sender;
 use tokio::net::TcpStream;
 use uuid::Uuid;
 
@@ -39,11 +46,18 @@ impl NodeLogger<'_> {
             .log(level, Some(self.node_id.clone()), target, message)
             .await
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<NodeLogger<'static>> {
+        Ok(NodeLogger {
+            node_id: self.node_id.clone(),
+            logger: self.logger.try_clone().await?,
+        })
+    }
 }
 
 pub struct DataflowLogger<'a> {
     dataflow_id: Uuid,
-    logger: &'a mut DaemonLogger,
+    logger: CowMut<'a, DaemonLogger>,
 }
 
 impl<'a> DataflowLogger<'a> {
@@ -57,12 +71,12 @@ impl<'a> DataflowLogger<'a> {
     pub fn reborrow(&mut self) -> DataflowLogger {
         DataflowLogger {
             dataflow_id: self.dataflow_id,
-            logger: self.logger,
+            logger: CowMut::Borrowed(&mut self.logger),
         }
     }
 
     pub fn inner(&self) -> &DaemonLogger {
-        self.logger
+        &self.logger
     }
 
     pub async fn log(
@@ -73,9 +87,64 @@ impl<'a> DataflowLogger<'a> {
         message: impl Into<String>,
     ) {
         self.logger
-            .log(level, self.dataflow_id, node_id, target, message)
+            .log(level, Some(self.dataflow_id), node_id, target, message)
             .await
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<DataflowLogger<'static>> {
+        Ok(DataflowLogger {
+            dataflow_id: self.dataflow_id,
+            logger: CowMut::Owned(self.logger.try_clone().await?),
+        })
+    }
+}
+
+pub struct NodeBuildLogger<'a> {
+    build_id: BuildId,
+    node_id: NodeId,
+    logger: CowMut<'a, DaemonLogger>,
+}
+
+impl NodeBuildLogger<'_> {
+    pub async fn log(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String>,
+    ) {
+        self.logger
+            .log_build(
+                self.build_id,
+                level.into(),
+                None,
+                Some(self.node_id.clone()),
+                message,
+            )
+            .await
+    }
+
+    pub async fn try_clone_impl(&self) -> eyre::Result<NodeBuildLogger<'static>> {
+        Ok(NodeBuildLogger {
+            build_id: self.build_id,
+            node_id: self.node_id.clone(),
+            logger: CowMut::Owned(self.logger.try_clone().await?),
+        })
+    }
+}
+
+impl BuildLogger for NodeBuildLogger<'_> {
+    type Clone = NodeBuildLogger<'static>;
+
+    fn log_message(
+        &mut self,
+        level: impl Into<LogLevelOrStdout> + Send,
+        message: impl Into<String> + Send,
+    ) -> impl std::future::Future<Output = ()> + Send {
+        self.log(level, message)
+    }
+
+    fn try_clone(&self) -> impl std::future::Future<Output = eyre::Result<Self::Clone>> + Send {
+        self.try_clone_impl()
+    }
 }
 
 pub struct DaemonLogger {
@@ -87,7 +156,15 @@ impl DaemonLogger {
     pub fn for_dataflow(&mut self, dataflow_id: Uuid) -> DataflowLogger {
         DataflowLogger {
             dataflow_id,
-            logger: self,
+            logger: CowMut::Borrowed(self),
+        }
+    }
+
+    pub fn for_node_build(&mut self, build_id: BuildId, node_id: NodeId) -> NodeBuildLogger {
+        NodeBuildLogger {
+            build_id,
+            node_id,
+            logger: CowMut::Borrowed(self),
         }
     }
 
@@ -98,15 +175,39 @@ impl DaemonLogger {
     pub async fn log(
         &mut self,
         level: LogLevel,
-        dataflow_id: Uuid,
+        dataflow_id: Option<Uuid>,
         node_id: Option<NodeId>,
         target: Option<String>,
         message: impl Into<String>,
     ) {
         let message = LogMessage {
+            build_id: None,
             daemon_id: Some(self.daemon_id.clone()),
             dataflow_id,
             node_id,
+            level: level.into(),
+            target,
+            module_path: None,
+            file: None,
+            line: None,
+            message: message.into(),
+        };
+        self.logger.log(message).await
+    }
+
+    pub async fn log_build(
+        &mut self,
+        build_id: BuildId,
+        level: LogLevelOrStdout,
+        target: Option<String>,
+        node_id: Option<NodeId>,
+        message: impl Into<String>,
+    ) {
+        let message = LogMessage {
+            build_id: Some(build_id),
+            daemon_id: Some(self.daemon_id.clone()),
+            dataflow_id: None,
+            node_id,
             level,
             target,
             module_path: None,
@@ -120,10 +221,17 @@ impl DaemonLogger {
     pub(crate) fn daemon_id(&self) -> &DaemonId {
         &self.daemon_id
     }
+
+    pub async fn try_clone(&self) -> eyre::Result<Self> {
+        Ok(Self {
+            daemon_id: self.daemon_id.clone(),
+            logger: self.logger.try_clone().await?,
+        })
+    }
 }
 
 pub struct Logger {
-    pub(super) coordinator_connection: Option<TcpStream>,
+    pub(super) destination: LogDestination,
     pub(super) daemon_id: DaemonId,
     pub(super) clock: Arc<uhlc::HLC>,
 }
@@ -137,73 +245,179 @@ impl Logger {
     }
 
     pub async fn log(&mut self, message: LogMessage) {
-        if let Some(connection) = &mut self.coordinator_connection {
-            let msg = serde_json::to_vec(&Timestamped {
-                inner: CoordinatorRequest::Event {
-                    daemon_id: self.daemon_id.clone(),
-                    event: DaemonEvent::Log(message.clone()),
-                },
-                timestamp: self.clock.new_timestamp(),
-            })
-            .expect("failed to serialize log message");
-            match socket_stream_send(connection, &msg)
-                .await
-                .wrap_err("failed to send log message to dora-coordinator")
-            {
-                Ok(()) => return,
-                Err(err) => tracing::warn!("{err:?}"),
+        match &mut self.destination {
+            LogDestination::Coordinator {
+                coordinator_connection,
+            } => {
+                let message = Timestamped {
+                    inner: CoordinatorRequest::Event {
+                        daemon_id: self.daemon_id.clone(),
+                        event: DaemonEvent::Log(message.clone()),
+                    },
+                    timestamp: self.clock.new_timestamp(),
+                };
+                Self::log_to_coordinator(message, coordinator_connection).await
             }
-        }
-
-        // log message using tracing if reporting to coordinator is not possible
-        match message.level {
-            LogLevel::Error => {
-                if let Some(node_id) = message.node_id {
-                    tracing::error!("{}/{} errored:", message.dataflow_id.to_string(), node_id);
-                }
-                for line in message.message.lines() {
-                    tracing::error!("   {}", line);
-                }
+            LogDestination::Channel { sender } => {
+                let _ = sender.send_async(message).await;
             }
-            LogLevel::Warn => {
-                if let Some(node_id) = message.node_id {
-                    tracing::warn!("{}/{} warned:", message.dataflow_id.to_string(), node_id);
-                }
-                for line in message.message.lines() {
-                    tracing::warn!("    {}", line);
+            LogDestination::Tracing => {
+                // log message using tracing if reporting to coordinator is not possible
+                match message.level {
+                    LogLevelOrStdout::Stdout => {
+                        tracing::info!(
+                            build_id = ?message.build_id.map(|id| id.to_string()),
+                            dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                            node_id = ?message.node_id.map(|id| id.to_string()),
+                            target = message.target,
+                            module_path = message.module_path,
+                            file = message.file,
+                            line = message.line,
+                            "{}",
+                            Indent(&message.message)
+                        )
+                    }
+                    LogLevelOrStdout::LogLevel(level) => match level {
+                        LogLevel::Error => {
+                            tracing::error!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Warn => {
+                            tracing::warn!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Info => {
+                            tracing::info!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        LogLevel::Debug => {
+                            tracing::debug!(
+                                build_id = ?message.build_id.map(|id| id.to_string()),
+                                dataflow_id = ?message.dataflow_id.map(|id| id.to_string()),
+                                node_id = ?message.node_id.map(|id| id.to_string()),
+                                target = message.target,
+                                module_path = message.module_path,
+                                file = message.file,
+                                line = message.line,
+                                "{}",
+                                Indent(&message.message)
+                            );
+                        }
+                        _ => {}
+                    },
                 }
             }
-            LogLevel::Info => {
-                if let Some(node_id) = message.node_id {
-                    tracing::info!("{}/{} info:", message.dataflow_id.to_string(), node_id);
-                }
-
-                for line in message.message.lines() {
-                    tracing::info!("    {}", line);
-                }
-            }
-            _ => {}
         }
     }
 
     pub async fn try_clone(&self) -> eyre::Result<Self> {
-        let coordinator_connection = match &self.coordinator_connection {
-            Some(c) => {
-                let addr = c
+        let destination = match &self.destination {
+            LogDestination::Coordinator {
+                coordinator_connection,
+            } => {
+                let addr = coordinator_connection
                     .peer_addr()
                     .context("failed to get coordinator peer addr")?;
                 let new_connection = TcpStream::connect(addr)
                     .await
                     .context("failed to connect to coordinator during logger clone")?;
-                Some(new_connection)
+                LogDestination::Coordinator {
+                    coordinator_connection: new_connection,
+                }
             }
-            None => None,
+            LogDestination::Channel { sender } => LogDestination::Channel {
+                sender: sender.clone(),
+            },
+            LogDestination::Tracing => LogDestination::Tracing,
         };
 
         Ok(Self {
-            coordinator_connection,
+            destination,
             daemon_id: self.daemon_id.clone(),
             clock: self.clock.clone(),
         })
     }
+
+    async fn log_to_coordinator(
+        message: Timestamped<CoordinatorRequest>,
+        connection: &mut TcpStream,
+    ) {
+        let msg = serde_json::to_vec(&message).expect("failed to serialize log message");
+        match socket_stream_send(connection, &msg)
+            .await
+            .wrap_err("failed to send log message to dora-coordinator")
+        {
+            Ok(()) => return,
+            Err(err) => tracing::warn!("{err:?}"),
+        }
+    }
+}
+
+pub enum LogDestination {
+    Coordinator { coordinator_connection: TcpStream },
+    Channel { sender: Sender<LogMessage> },
+    Tracing,
+}
+
+enum CowMut<'a, T> {
+    Borrowed(&'a mut T),
+    Owned(T),
+}
+
+impl<T> Deref for CowMut<'_, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            CowMut::Borrowed(v) => v,
+            CowMut::Owned(v) => v,
+        }
+    }
+}
+
+impl<T> DerefMut for CowMut<'_, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        match self {
+            CowMut::Borrowed(v) => v,
+            CowMut::Owned(v) => v,
+        }
+    }
+}
+
+struct Indent<'a>(&'a str);
+
+impl std::fmt::Display for Indent<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for line in self.0.lines() {
+            write!(f, "   {}", line)?;
+        }
+        Ok(())
+    }
 }

binaries/daemon/src/pending.rs

@@ -59,6 +59,10 @@ impl PendingNodes {
         self.external_nodes = value;
     }
 
+    pub fn local_nodes_pending(&self) -> bool {
+        !self.local_nodes.is_empty()
+    }
+
     pub async fn handle_node_subscription(
         &mut self,
         node_id: NodeId,

binaries/runtime/Cargo.toml

@@ -21,7 +21,7 @@ eyre = "0.6.8"
 futures = "0.3.21"
 futures-concurrency = "7.1.0"
 libloading = "0.7.3"
-serde_yaml = "0.8.23"
+serde_yaml = { workspace = true }
 tokio = { version = "1.24.2", features = ["full"] }
 tokio-stream = "0.1.8"
 # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html

binaries/runtime/src/lib.rs

@@ -43,7 +43,8 @@ pub fn main() -> eyre::Result<()> {
             .wrap_err("failed to set up tracing subscriber")?;
     }
 
-    let dataflow_descriptor = config.dataflow_descriptor.clone();
+    let dataflow_descriptor = serde_yaml::from_value(config.dataflow_descriptor.clone())
+        .context("failed to parse dataflow descriptor")?;
 
     let operator_definition = if operators.is_empty() {
         bail!("no operators");
@@ -232,10 +233,10 @@ async fn run(
                     }
                 }
             }
-            RuntimeEvent::Event(Event::Stop) => {
+            RuntimeEvent::Event(Event::Stop(cause)) => {
                 // forward stop event to all operators and close the event channels
                 for (_, channel) in operator_channels.drain() {
-                    let _ = channel.send_async(Event::Stop).await;
+                    let _ = channel.send_async(Event::Stop(cause.clone())).await;
                 }
             }
             RuntimeEvent::Event(Event::Reload {

binaries/runtime/src/operator/shared_lib.rs

@@ -182,7 +182,7 @@ impl<'lib> SharedLibraryOperator<'lib> {
             }
 
             let mut operator_event = match event {
-                Event::Stop => dora_operator_api_types::RawEvent {
+                Event::Stop(_) => dora_operator_api_types::RawEvent {
                     input: None,
                     input_closed: None,
                     stop: true,

examples/c++-arrow-dataflow/run.rs

@@ -112,6 +112,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -136,6 +137,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]

examples/c++-dataflow/.gitignore

@@ -1 +1,2 @@
 *.o
+/build

examples/c++-dataflow/run.rs

@@ -133,6 +133,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -157,6 +158,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]

examples/c++-ros2-dataflow/.gitignore

@@ -1 +1,2 @@
 *.o
+/build

examples/c++-ros2-dataflow/run.rs

@@ -90,6 +90,7 @@ async fn build_cxx_node(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]
@@ -154,6 +155,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/c-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")
@@ -63,6 +64,7 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-l").arg("m");
         clang.arg("-l").arg("rt");
         clang.arg("-l").arg("dl");
+        clang.arg("-l").arg("z");
         clang.arg("-pthread");
     }
     #[cfg(target_os = "windows")]
@@ -93,6 +95,8 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-lsynchronization");
         clang.arg("-luser32");
         clang.arg("-lwinspool");
+        clang.arg("-lwinhttp");
+        clang.arg("-lrpcrt4");
 
         clang.arg("-Wl,-nodefaultlib:libcmt");
         clang.arg("-D_DLL");
@@ -107,6 +111,7 @@ async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<(
         clang.arg("-l").arg("pthread");
         clang.arg("-l").arg("c");
         clang.arg("-l").arg("m");
+        clang.arg("-l").arg("z");
     }
     clang.arg("-L").arg(root.join("target").join("debug"));
     clang
@@ -161,6 +166,8 @@ async fn build_c_operator(root: &Path) -> eyre::Result<()> {
         link.arg("-lsynchronization");
         link.arg("-luser32");
         link.arg("-lwinspool");
+        link.arg("-lwinhttp");
+        link.arg("-lrpcrt4");
 
         link.arg("-Wl,-nodefaultlib:libcmt");
         link.arg("-D_DLL");

examples/camera/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/cmake-dataflow/run.rs

@@ -61,6 +61,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/mujoco-sim/README.md

@@ -0,0 +1,21 @@
+# MuJoCo Sim Tutorial
+
+This comprehensive tutorial demonstrates how to build a robot control system using Dora with the `dora-mujoco` simulation node and control logic.
+
+## Tutorial Structure
+
+### [01. Basic Simulation](basic_simulation/)
+Load a robot in simulation using the `dora-mujoco` node.
+- Learn the fundamentals of MuJoCo simulation in Dora
+- Understand the simulation node architecture
+- See how robot descriptions are loaded automatically
+
+### [02. Target Pose Control](target_pose_control/) 
+Add control logic with pose commands as target.
+- Implement Cartesian space control.
+- Create generic controller node that is able to control any robotic arm by using `dora-pytorch-kinematics`
+
+### [03. Gamepad Control](gamepad_control/)
+Connect a gamepad for real-time interactive control.
+- Integrate with dora's `gamepad` node
+- Demonstrate real-time teleoperation

examples/mujoco-sim/basic_simulation/README.md

@@ -0,0 +1,40 @@
+# 01. Basic Simulation
+
+This example demonstrates the simplest possible setup: loading and running a robot simulation using the `dora-mujoco` node.
+
+- Understand how the `dora-mujoco` node works
+- See how robot models are loaded from `robot-descriptions`
+- Learn the basic dataflow for physics simulation
+
+
+The simulation runs at 500Hz and outputs:
+- Joint positions for all robot joints
+- Joint velocities 
+- Sensor data (if available)
+- Current simulation time
+
+### Running the Example
+
+```bash
+cd basic_simulation
+dora build basic.yml
+dora run basic.yml
+```
+
+You should see:
+1. MuJoCo viewer window opens with GO2 robot
+2. Robot is effected by gravity (enabled by default)
+
+### What's Happening
+
+1. **Model Loading**: The `dora-mujoco` node loads the RoboDog (go2) model using `load_robot_description("go2_mj_description")`
+2. **Physics Loop**: Timer triggers simulation steps at 500Hz (This is default step time for Mujoco)
+3. **Data Output**: Joint states are published 
+4. **Visualization**: MuJoCo viewer shows real-time simulation
+
+### Configuration Details
+
+The `basic.yml` configures:
+- Model name: `"go2"` you change this to other robots name
+- Update rate: 2ms (500Hz) 
+- Outputs: Joint positions, velocities, and sensor data

examples/mujoco-sim/basic_simulation/basic.yml

@@ -0,0 +1,12 @@
+nodes:
+  - id: mujoco_sim
+    build: pip install -e ../../../node-hub/dora-mujoco
+    path: dora-mujoco
+    inputs:
+      tick: dora/timer/millis/2 # 500 Hz simulation
+    outputs:
+      - joint_positions
+      - joint_velocities
+      - sensor_data
+    env:
+      MODEL_NAME: "go2_mj_description" # Load GO2

examples/mujoco-sim/gamepad_control/README.md

@@ -0,0 +1,134 @@
+# 03. Gamepad Control
+
+This example demonstrates real-time interactive control by connecting a gamepad to the controller. It builds upon the target pose control example by adding gamepad input processing for teleoperation of the robot arm.
+
+## Controller Types
+
+### 1. Basic Gamepad Control (`gamepad_control_basic.yml`)
+- **Direct IK approach**: Uses simple IK solver for immediate position updates
+- The movement fells jumpy
+
+### 2. Advanced Gamepad Control (`gamepad_control_advanced.yml`)
+- **Differential IK approach**: Smooth velocity-based control with Jacobian
+- **Smooth**: Continuous motion interpolation
+- **Use case**: Precise manipulation, smooth trajectories
+
+## Gamepad Controls
+
+- **D-pad Vertical**: Move along X-axis (forward/backward) 
+- **D-pad Horizontal**: Move along Y-axis (left/right)
+- **Right Stick Y**: Move along Z-axis (up/down)
+- **LB/RB**: Decrease/Increase movement speed (0.1-1.0x scale)
+- **START**: Reset to home position [0.4, 0.0, 0.3]
+
+## Running the Examples
+
+1. **Connect a gamepad** (Xbox/PlayStation controller via USB or Bluetooth)
+
+### Basic Gamepad Control
+```bash
+cd gamepad_control
+dora build gamepad_control_basic.yml
+dora run gamepad_control_basic.yml
+```
+
+### Advanced Gamepad Control
+```bash
+cd gamepad_control
+dora build gamepad_control_advanced.yml
+dora run gamepad_control_advanced.yml
+```
+
+You should see:
+1. Robot responds to gamepad input in real-time
+2. **Basic**: Immediate position jumps based on gamepad input
+3. **Advanced**: Smooth incremental movement with velocity control
+4. Speed scaling with bumper buttons
+5. Reset functionality with START button
+
+
+### **Gamepad Node** (`gamepad`)
+Built-in Dora node that interfaces with system gamepad drivers using Pygame.
+
+```yaml
+- id: gamepad
+  build: pip install -e ../../../node-hub/gamepad
+  path: gamepad
+  outputs:
+    - cmd_vel     # 6DOF velocity commands
+    - raw_control # Raw button/stick states
+  inputs:
+    tick: dora/timer/millis/10  # 100Hz polling
+```
+
+- **`cmd_vel`**: 6DOF velocity array `[linear_x, linear_y, linear_z, angular_x, angular_y, angular_z]`
+  - Generated from D-pad and analog stick positions
+  - Continuous updates while controls are held
+  
+- **`raw_control`**: JSON format gamepad state
+
+
+### **Gamepad Controller Scripts**
+
+**Basic Controller (`gamepad_controller_ik.py`)**:
+```
+Gamepad Input → Target Position Update → IK Request → Joint Commands
+```
+- Updates target position incrementally based on gamepad
+- Immediate position jumps
+
+**Advanced Controller (`gamepad_controller_differential_ik.py`)**:
+```
+Gamepad Input → Target Position → Pose Error → Velocity → Joint Commands
+```
+- Continuous pose error calculation and velocity control
+- Smooth interpolation between current and target poses
+- Real-time Jacobian-based control
+
+## Gamepad Mapping
+
+The controller expects standard gamepad layout: (The mapping may change based on controller)
+
+| Control | Function |
+|---------|----------|
+| D-pad Up/Down |  X-axis movement |
+| D-pad Left/Right |  Y-axis movement |
+| Right Stick Y |  Z-axis movement |
+| Left Bumper |  Decrease speed |
+| Right Bumper |  Increase speed |
+| START button |  Reset position |
+
+## Key Features
+
+**Real-time Teleoperation:**
+- Incremental position updates based on continuous input
+- Immediate feedback through robot motion
+
+**Speed Control:**
+- Dynamic speed scaling (0.1x to 1.0x)
+- Allows both coarse and fine manipulation
+
+**Features:**
+- Home position reset capability
+- Bounded movement through incremental updates
+- Working on collision avoidance
+
+## Troubleshooting
+
+**"Gamepad not responding"**
+```bash
+# Check if gamepad is connected
+ls /dev/input/js*
+# Test gamepad input  
+jstest /dev/input/js0
+# Grant permissions
+sudo chmod 666 /dev/input/js0
+```
+
+**"Robot doesn't move with D-pad"**
+- Check `cmd_vel` output: should show non-zero values when D-pad pressed
+- Verify correct gamepad mapping for your controller model
+
+**"Movement too fast/slow"**
+- Use LB/RB buttons to adjust speed scale
+- Modify `delta = cmd_vel[:3] * 0.03` scaling factor in code if required 

examples/mujoco-sim/gamepad_control/gamepad_control_advanced.yml

@@ -0,0 +1,51 @@
+nodes:
+  - id: gamepad
+    build: pip install -e ../../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+
+  - id: mujoco_sim
+    build: pip install -e ../../../node-hub/dora-mujoco
+    path: dora-mujoco
+    inputs:
+      tick: dora/timer/millis/2 # 500 Hz simulation
+      control_input: gamepad_controller/joint_commands
+    outputs:
+      - joint_positions
+      - joint_velocities
+      - actuator_controls
+      - sensor_data
+    env:
+      MODEL_NAME: "iiwa14_mj_description"
+
+  - id: gamepad_controller
+    path: nodes/gamepad_controller_differential_ik.py
+    inputs:
+      joint_positions: mujoco_sim/joint_positions
+      joint_velocities: mujoco_sim/joint_velocities
+      raw_control: gamepad/raw_control
+      cmd_vel: gamepad/cmd_vel
+      fk_result: pytorch_kinematics/fk_request
+      jacobian_result: pytorch_kinematics/jacobian_request
+    outputs:
+      - joint_commands
+      - fk_request
+      - jacobian_request
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      fk_request: gamepad_controller/fk_request
+      jacobian_request: gamepad_controller/jacobian_request
+    outputs:
+      - fk_request
+      - jacobian_request
+    env:
+      MODEL_NAME: "iiwa14_description"
+      END_EFFECTOR_LINK: "iiwa_link_7"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/mujoco-sim/gamepad_control/gamepad_control_basic.yml

@@ -0,0 +1,37 @@
+nodes:
+  - id: gamepad
+    build: pip install -e ../../../node-hub/gamepad
+    path: gamepad
+    outputs:
+      - cmd_vel
+      - raw_control
+    inputs:
+      tick: dora/timer/millis/10
+
+  - id: mujoco_sim
+    build: pip install -e ../../../node-hub/dora-mujoco
+    path: dora-mujoco
+    inputs:
+      tick: dora/timer/millis/2 # 500 Hz simulation
+      control_input: pytorch_kinematics/cmd_vel
+    outputs:
+      - joint_positions
+      - joint_velocities
+      - actuator_controls
+      - sensor_data
+    env:
+      MODEL_NAME: "iiwa14_mj_description"
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      cmd_vel: gamepad/cmd_vel
+      pose: mujoco_sim/joint_positions
+    outputs:
+      - cmd_vel
+      - pose
+    env:
+      MODEL_NAME: "iiwa14_description"
+      END_EFFECTOR_LINK: "iiwa_link_7"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0."

examples/mujoco-sim/gamepad_control/nodes/gamepad_controller_differential_ik.py

@@ -0,0 +1,188 @@
+import json
+import time
+import numpy as np
+import pyarrow as pa
+from dora import Node
+from scipy.spatial.transform import Rotation
+
+class GamepadController:
+    def __init__(self):
+        # Control parameters
+        self.integration_dt = 0.1
+        self.damping = 1e-4
+        self.Kpos = 0.95  # Position gain
+        self.Kori = 0.95  # Orientation gain
+
+        # Gamepad control parameters
+        self.speed_scale = 0.5
+        self.max_angvel = 3.14 * self.speed_scale  # Max joint velocity (rad/s)
+        
+        # Robot state variables
+        self.dof = None
+        self.current_joint_pos = None  # Full robot state
+        self.home_pos = None  # Home position for arm joints only
+        
+        # Target pose (independent of DOF)
+        self.target_pos = np.array([0.4, 0.0, 0.3])  # Conservative default target
+        self.target_rpy = [180.0, 0.0, 90.0]  # Downward orientation
+        
+        # Cache for external computations
+        self.current_ee_pose = None  # End-effector pose
+        self.current_jacobian = None  # Jacobian matrix
+        
+        print("Gamepad Controller initialized")
+    
+    def _initialize_robot(self, positions, jacobian_dof=None):
+        self.full_joint_count = len(positions)
+        self.dof = jacobian_dof if jacobian_dof is not None else self.full_joint_count
+        self.current_joint_pos = positions.copy()
+        self.home_pos = np.zeros(self.dof)
+    
+    def process_cmd_vel(self, cmd_vel):
+        # print(f"Processing cmd_vel: {cmd_vel}")
+        delta = cmd_vel[:3] * 0.03
+        dx, dy, dz = delta
+        
+        # Update target position incrementally
+        if abs(dx) > 0 or abs(dy) > 0 or abs(dz) > 0:
+            self.target_pos += np.array([dx, -dy, dz])
+    
+    def process_gamepad_input(self, raw_control):
+        buttons = raw_control["buttons"]
+
+        # Reset position with START button
+        if len(buttons) > 9 and buttons[9]:
+            self.target_pos = np.array([0.4, 0.0, 0.3])
+            print("Reset target to home position")
+        
+        # Speed scaling with bumpers (LB/RB)
+        if len(buttons) > 4 and buttons[4]:  # LB
+            self.speed_scale = max(0.1, self.speed_scale - 0.1)
+            print(f"Speed: {self.speed_scale:.1f}")
+        if len(buttons) > 5 and buttons[5]:  # RB
+            self.speed_scale = min(1.0, self.speed_scale + 0.1)
+            print(f"Speed: {self.speed_scale:.1f}")
+    
+    def get_target_rotation_matrix(self):
+        roll_rad, pitch_rad, yaw_rad = np.radians(self.target_rpy)
+        desired_rot = Rotation.from_euler('ZYX', [yaw_rad, pitch_rad, roll_rad])
+        return desired_rot.as_matrix()
+    
+    def update_jacobian(self, jacobian_flat, shape):
+        jacobian_dof = shape[1]
+        self.current_jacobian = jacobian_flat.reshape(shape)
+
+        if self.dof is None:
+            if self.current_joint_pos is not None:
+                self._initialize_robot(self.current_joint_pos, jacobian_dof)
+            else:
+                self.dof = jacobian_dof
+        elif self.dof != jacobian_dof:
+            self.dof = jacobian_dof
+            self.home_pos = np.zeros(self.dof)
+    
+    def apply_differential_ik_control(self):
+        if self.current_ee_pose is None or self.current_jacobian is None:
+            return self.current_joint_pos
+            
+        current_ee_pos = self.current_ee_pose['position']
+        current_ee_rpy = self.current_ee_pose['rpy']
+
+        pos_error = self.target_pos - current_ee_pos
+        twist = np.zeros(6)
+        twist[:3] = self.Kpos * pos_error / self.integration_dt
+        
+        # Convert current RPY to rotation matrix
+        current_rot = Rotation.from_euler('XYZ', current_ee_rpy)
+        desired_rot = Rotation.from_matrix(self.get_target_rotation_matrix())
+        rot_error = (desired_rot * current_rot.inv()).as_rotvec()
+        twist[3:] = self.Kori * rot_error / self.integration_dt
+        
+        jac = self.current_jacobian
+        
+        # Damped least squares inverse kinematics
+        diag = self.damping * np.eye(6)
+        dq = jac.T @ np.linalg.solve(jac @ jac.T + diag, twist)
+        
+        # # Apply nullspace control
+        # current_arm = self.current_joint_pos[:self.dof]
+        # jac_pinv = np.linalg.pinv(jac)
+        # N = np.eye(self.dof) - jac_pinv @ jac
+        # k_null = np.ones(self.dof) * 5.0
+        # dq_null = k_null * (self.home_pos - current_arm)
+        # dq += N @ dq_null
+        
+        # Limit joint velocities
+        dq_abs_max = np.abs(dq).max()
+        if dq_abs_max > self.max_angvel:
+            dq *= self.max_angvel / dq_abs_max
+            
+        # Create full joint command
+        new_joints = self.current_joint_pos.copy()
+        new_joints[:self.dof] += dq * self.integration_dt
+        
+        return new_joints
+
+
+def main():
+    node = Node()
+    controller = GamepadController()
+    
+    for event in node:
+        if event["type"] == "INPUT":
+            
+            if event["id"] == "joint_positions":
+                joint_pos = event["value"].to_numpy()
+                
+                if controller.current_joint_pos is None:
+                    controller._initialize_robot(joint_pos)
+                else:
+                    controller.current_joint_pos = joint_pos
+                
+                # Request FK computation
+                node.send_output(
+                    "fk_request", 
+                    pa.array(controller.current_joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jointstate", "timestamp": time.time()}
+                )
+                
+                # Request Jacobian computation
+                node.send_output(
+                    "jacobian_request", 
+                    pa.array(controller.current_joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jacobian", "timestamp": time.time()}
+                )
+                
+                # Apply differential IK control
+                joint_commands = controller.apply_differential_ik_control()
+                
+                # Send control commands to the robot
+                node.send_output(
+                    "joint_commands", 
+                    pa.array(joint_commands, type=pa.float32()),
+                    metadata={"timestamp": time.time()}
+                )
+                
+            elif event["id"] == "raw_control":
+                raw_control = json.loads(event["value"].to_pylist()[0])
+                controller.process_gamepad_input(raw_control)
+                
+            elif event["id"] == "cmd_vel":
+                cmd_vel = event["value"].to_numpy()
+                controller.process_cmd_vel(cmd_vel)
+            
+            # Handle FK results
+            if event["id"] == "fk_result":
+                if event["metadata"].get("encoding") == "xyzrpy":
+                    ee_pose = event["value"].to_numpy()
+                    controller.current_ee_pose = {'position': ee_pose[:3], 'rpy': ee_pose[3:6]}
+                    
+            # Handle Jacobian results
+            if event["id"] == "jacobian_result":
+                if event["metadata"].get("encoding") == "jacobian_result":
+                    jacobian_flat = event["value"].to_numpy()
+                    jacobian_shape = event["metadata"]["jacobian_shape"]
+                    controller.update_jacobian(jacobian_flat, jacobian_shape)
+
+if __name__ == "__main__":
+    main()

examples/mujoco-sim/gamepad_control/nodes/gamepad_controller_ik.py

@@ -0,0 +1,106 @@
+import json
+import time
+import numpy as np
+import pyarrow as pa
+from dora import Node
+
+class GamepadController:
+    def __init__(self):
+        """
+        Initialize the simple gamepad controller
+        """
+        # Robot state variables
+        self.dof = None 
+        self.current_joint_pos = None
+        
+        # Target pose (independent of DOF)
+        self.target_pos = np.array([0.4, 0.0, 0.3])  # Conservative default target
+        self.target_rpy = [180.0, 0.0, 90.0]  # Downward orientation
+        self.ik_request_sent = True 
+
+        
+        print("Simple Gamepad Controller initialized")
+    
+    def _initialize_robot(self, positions):
+        """Initialize controller state with appropriate dimensions"""
+        self.full_joint_count = len(positions)
+        self.current_joint_pos = positions.copy()
+        if self.dof is None:
+            self.dof = self.full_joint_count
+    
+    def process_cmd_vel(self, cmd_vel):
+        """Process gamepad velocity commands to update target position"""
+        delta = cmd_vel[:3] * 0.03
+        dx, dy, dz = delta
+        
+        # Update target position incrementally
+        if abs(dx) > 0 or abs(dy) > 0 or abs(dz) > 0:
+            self.target_pos += np.array([dx, -dy, dz])
+            self.ik_request_sent = False 
+
+    
+    def process_gamepad_input(self, raw_control):
+        """Process gamepad button inputs"""
+        buttons = raw_control["buttons"]
+
+        # Reset position with START button
+        if len(buttons) > 9 and buttons[9]:
+            self.target_pos = np.array([0.4, 0.0, 0.3])
+            print("Reset target to home position")
+            self.ik_request_sent = False 
+    
+    def get_target_pose_array(self):
+        """Get target pose as 6D array [x, y, z, roll, pitch, yaw]"""
+        return np.concatenate([self.target_pos, self.target_rpy])
+
+def main():
+    node = Node()
+    controller = GamepadController()
+    
+    for event in node:
+        if event["type"] == "INPUT":
+            if event["id"] == "joint_positions":
+                joint_pos = event["value"].to_numpy()
+
+                if controller.current_joint_pos is None:
+                    controller._initialize_robot(joint_pos)
+                else:
+                    controller.current_joint_pos = joint_pos
+                
+                # Request IK solution directly
+                target_pose = controller.get_target_pose_array()
+                if not controller.ik_request_sent:
+                    node.send_output(
+                        "ik_request", 
+                        pa.array(target_pose, type=pa.float32()),
+                        metadata={"encoding": "xyzrpy", "timestamp": time.time()}
+                    )
+                    controller.ik_request_sent = True 
+
+                node.send_output(
+                    "joint_state", 
+                    pa.array(joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jointstate", "timestamp": time.time()}
+                )
+                
+            elif event["id"] == "raw_control":
+                raw_control = json.loads(event["value"].to_pylist()[0])
+                controller.process_gamepad_input(raw_control)
+                
+            elif event["id"] == "cmd_vel":
+                cmd_vel = event["value"].to_numpy()
+                controller.process_cmd_vel(cmd_vel)
+            
+            # Handle IK results and send directly as joint commands
+            elif event["id"] == "ik_request":
+                if event["metadata"]["encoding"] == "jointstate":
+                    ik_solution = event["value"].to_numpy()
+                    # Send IK solution directly as joint commands
+                    node.send_output(
+                        "joint_commands", 
+                        pa.array(ik_solution, type=pa.float32()),
+                        metadata={"timestamp": time.time()}
+                    )
+
+if __name__ == "__main__":
+    main()

examples/mujoco-sim/target_pose_control/README.md

@@ -0,0 +1,167 @@
+# 02. Target Pose Control
+
+This example demonstrates Cartesian space control using two different approaches: **Direct Inverse Kinematics(IK)**(Basic) and **Differential IK**(advance). Both create a Controller node that processes target pose commands and outputs joint commands using **dora-pytorch-kinematics**.
+
+## Controller Types
+
+### 1. Direct IK Controller (`control.yml`)
+- **Simple approach**: Directly passes target pose to IK solver
+- **Fast**: Single IK computation per target pose
+
+### 2. Differential IK Controller (`control_advanced.yml`)  
+- **Smooth approach**: Uses pose error feedback and Jacobian-based control
+- **Continuous**: Interpolates smoothly between current and target poses
+- **Use case**: Smooth trajectories
+
+## Running the Examples
+
+### Direct IK Control
+```bash
+cd target_pose_control
+dora build control.yml
+dora run control.yml
+```
+
+### Differential IK Control
+```bash
+cd target_pose_control
+dora build control_advanced.yml
+dora run control_advanced.yml
+```
+
+You should see:
+1. Robot moves to predefined target poses automatically
+2. **Direct IK**: Immediate jumps to target poses
+3. **Differential IK**: Smooth Cartesian space motion with continuous interpolation
+4. End-effector following target positions accurately
+
+
+### Nodes
+
+#### 1. **Pose Publisher Script** (`pose_publisher.py`)
+```python
+class PosePublisher:
+    def __init__(self):
+        # Predefined sequence of target poses [x, y, z, roll, pitch, yaw]
+        self.target_poses = [
+            [0.5, 0.5, 0.3, 180.0, 0.0, 90.0],   # Position + RPY orientation
+            [0.6, 0.2, 0.5, 180.0, 0.0, 45.0],   # Different orientation
+            # ... more poses
+        ]
+```
+
+- Sends target poses every 5 seconds
+- Cycles through predefined positions and orientations
+- Can be replaced with path planning, user input, or any pose generation logic
+- Outputs `target_pose` array `[x, y, z, roll, pitch, yaw]` 
+
+#### 2. **Controller Scripts**
+
+##### Direct IK Controller (`controller_ik.py`)
+
+**How it works:**
+1. **Target Input**: Receives new target pose `[x, y, z, roll, pitch, yaw]`
+2. **IK Request**: Sends target pose directly to `dora-pytorch-kinematics`
+3. **Joint Solution**: Receives complete joint configuration for target pose
+4. **Direct Application**: Passes IK solution directly as joint commands to robot *(sometimes for certain target pose there is no IK solution)*
+
+**Advantages:**
+- Simple and fast0
+- Minimal computation
+- Direct pose-to-joint mapping
+
+**Disadvantages:**
+- Sudden jumps between poses
+- No trajectory smoothing
+- May cause joint velocity spikes 
+
+##### Differential IK Controller (`controller_differential_ik.py`)
+
+**How it works:**
+1. **Pose Error Calculation**: Computes difference between target and current end-effector pose
+2. **Velocity Command**: Converts pose error to desired end-effector velocity using PD control:
+   ```python
+   pos_error = target_pos - current_ee_pos
+   twist[:3] = Kpos * pos_error / integration_dt  # Linear velocity
+   twist[3:] = Kori * rot_error / integration_dt  # Angular velocity
+   ```
+3. **Jacobian Inverse**: Uses robot Jacobian to map end-effector velocity to joint velocities:
+   ```python
+   # Damped least squares to avoid singularities
+   dq = J^T @ (J @ J^T + λI)^(-1) @ twist
+   ```
+4. **Interpolation**: Integrates joint velocities to get next joint positions:
+   ```python
+   new_joints = current_joints + dq * dt
+   ```
+5. **Nullspace Control** (optional): Projects secondary objectives (like joint limits avoidance) into the nullspace
+
+
+**Advantages:**
+- Smooth, continuous motion
+- Velocity-controlled approach
+- Handles robot singularities 
+- Real-time reactive control
+
+<!-- **Jacobian Role:**
+- **Forward Kinematics**: Maps joint space to Cartesian space
+- **Jacobian Matrix**: Linear mapping between joint velocities and end-effector velocities
+- **Inverse Mapping**: Converts desired end-effector motion to required joint motions
+- **Singularity Handling**: Damped least squares prevents numerical instability near singularities -->
+
+##### 3. **PyTorch Kinematics Node** (`dora-pytorch-kinematics`)
+
+A dedicated kinematics computation node that provides three core robotic functions:
+
+```yaml
+- id: pytorch_kinematics
+  build: pip install -e ../../../node-hub/dora-pytorch-kinematics
+  path: dora-pytorch-kinematics
+  inputs:
+    ik_request: controller/ik_request           # For inverse kinematics
+    fk_request: controller/fk_request           # For forward kinematics  
+    jacobian_request: controller/jacobian_request  # For Jacobian computation
+  outputs:
+    - ik_request      # Joint solution for target pose
+    - fk_request      # End-effector pose for joint configuration
+    - jacobian_request # Jacobian matrix for velocity mapping
+  env:
+    URDF_PATH: "../URDF/franka_panda/panda.urdf"
+    END_EFFECTOR_LINK: "panda_hand"
+    TRANSFORM: "0. 0. 0. 1. 0. 0. 0."
+```
+
+1. **Inverse Kinematics (IK)**
+   - **Input**: Target pose `[x, y, z, roll, pitch, yaw]` or `[x, y, z, qw, qx, qy, qz]` + current joint state
+   - **Output**: Complete joint configuration to achieve target pose
+   - **Use case**: Convert Cartesian target to joint angles
+
+2. **Forward Kinematics (FK)**  
+   - **Input**: Joint positions array
+   - **Output**: Current end-effector pose `[x, y, z, qw, qx, qy, qz]`
+   - **Use case**: Determine end-effector position from joint angles
+
+3. **Jacobian Computation**
+   - **Input**: Current joint positions  
+   - **Output**: 6×N Jacobian matrix (N = number of joints)
+   - **Use case**: Map joint velocities to end-effector velocities
+
+**Configuration:**
+- **URDF_PATH**: Robot model definition file
+- **END_EFFECTOR_LINK**: Target link for pose calculations
+- **TRANSFORM**: Optional transform offset (position + quaternion wxyz format)
+
+**Usage in Controllers:**
+- **Direct IK**: Uses only `ik_request` → `ik_result`
+- **Differential IK**: Uses `fk_request` → `fk_result` and `jacobian_request` → `jacobian_result`
+
+#### 4. **MuJoCo Simulation Node** (`dora-mujoco`)
+
+- **Process**: Physics simulation, dynamics integration, rendering
+- **Output**: Joint positions, velocities, sensor data
+
+## References
+
+This controller design draws inspiration from the kinematic control strategies presented in [mjctrl](https://github.com/kevinzakka/mjctrl), specifically the [differential ik control example](https://github.com/kevinzakka/mjctrl/blob/main/diffik.py).
+
+The URDF model for the robotic arms was sourced from the [PyBullet GitHub repository](https://github.com/bulletphysics/bullet3/tree/master/examples/pybullet/gym/pybullet_data). Or you could google search the robot and get its urdf.

examples/mujoco-sim/target_pose_control/control.yml

@@ -0,0 +1,46 @@
+nodes:
+  - id: mujoco_sim
+    build: pip install -e ../../../node-hub/dora-mujoco
+    path: dora-mujoco
+    inputs:
+      tick: dora/timer/millis/2
+      control_input: controller/joint_commands
+    outputs:
+      - joint_positions
+      - joint_velocities
+      - actuator_controls
+      - sensor_data
+    env:
+      MODEL_NAME: "panda_mj_description"
+
+  - id: controller
+    path: nodes/controller_ik.py
+    inputs:
+      joint_positions: mujoco_sim/joint_positions
+      target_pose: pose_publisher/target_pose
+      ik_request: pytorch_kinematics/ik_request
+    outputs:
+      - joint_commands
+      - ik_request
+      - joint_state
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      ik_request: controller/ik_request
+      joint_state: controller/joint_state
+    outputs:
+      - ik_request
+      - joint_state
+    env:
+      MODEL_NAME: "panda_description"
+      END_EFFECTOR_LINK: "panda_hand"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0." # Pytorch kinematics uses wxyz format for quaternion
+
+  - id: pose_publisher
+    path: nodes/pose_publisher.py
+    inputs:
+      tick: dora/timer/millis/5000
+    outputs:
+      - target_pose

examples/mujoco-sim/target_pose_control/control_advanced.yml

@@ -0,0 +1,48 @@
+nodes:
+  - id: mujoco_sim
+    build: pip install -e ../../../node-hub/dora-mujoco
+    path: dora-mujoco
+    inputs:
+      tick: dora/timer/millis/2
+      control_input: controller/joint_commands
+    outputs:
+      - joint_positions
+      - joint_velocities
+      - actuator_controls
+      - sensor_data
+    env:
+      MODEL_NAME: "panda_mj_description"
+
+  - id: controller
+    path: nodes/controller_differential_ik.py
+    inputs:
+      joint_positions: mujoco_sim/joint_positions
+      joint_velocities: mujoco_sim/joint_velocities
+      target_pose: pose_publisher/target_pose
+      fk_result: pytorch_kinematics/fk_request
+      jacobian_result: pytorch_kinematics/jacobian_request
+    outputs:
+      - joint_commands
+      - fk_request
+      - jacobian_request
+
+  - id: pytorch_kinematics
+    build: pip install -e ../../../node-hub/dora-pytorch-kinematics
+    path: dora-pytorch-kinematics
+    inputs:
+      fk_request: controller/fk_request
+      jacobian_request: controller/jacobian_request
+    outputs:
+      - fk_request
+      - jacobian_request
+    env:
+      MODEL_NAME: "panda_description"
+      END_EFFECTOR_LINK: "panda_hand"
+      TRANSFORM: "0. 0. 0. 1. 0. 0. 0." # Pytorch kinematics uses wxyz format for quaternion
+
+  - id: pose_publisher
+    path: nodes/pose_publisher.py
+    inputs:
+      tick: dora/timer/millis/5000
+    outputs:
+      - target_pose

examples/mujoco-sim/target_pose_control/nodes/controller_differential_ik.py

@@ -0,0 +1,171 @@
+import numpy as np
+import time
+import pyarrow as pa
+from dora import Node
+from scipy.spatial.transform import Rotation
+
+class Controller:
+    def __init__(self):
+        """
+        Initialize the controller
+        """
+        # Control parameters
+        self.integration_dt = 0.1
+        self.damping = 1e-4
+        self.Kpos = 0.95  # Position gain
+        self.Kori = 0.95  # Orientation gain
+        self.max_angvel = 3.14 # Max joint velocity (rad/s)
+        
+        # State variables
+        self.dof = None 
+        self.current_joint_pos = None  # Full robot state
+        self.home_pos = None  # Home position for arm joints only
+        
+        self.target_pos = np.array([0.4, 0.0, 0.3])  # Conservative default target
+        self.target_rpy = [180.0, 0.0, 90.0]  # Downward orientation
+        
+        self.current_ee_pose = None  # End-effector pose
+        self.current_jacobian = None  # Jacobian matrix
+        
+        print("Controller initialized")
+    
+    def _initialize_robot(self, positions, jacobian_dof=None):
+        """Initialize controller state with appropriate dimensions"""
+        self.full_joint_count = len(positions)
+        # Set DOF from Jacobian if available
+        self.dof = jacobian_dof if jacobian_dof is not None else self.full_joint_count
+
+        self.current_joint_pos = positions.copy()
+        self.home_pos = np.zeros(self.dof)
+    
+    def get_target_rotation_matrix(self):
+        """Convert RPY to rotation matrix."""
+        roll_rad, pitch_rad, yaw_rad = np.radians(self.target_rpy)
+        desired_rot = Rotation.from_euler('ZYX', [yaw_rad, pitch_rad, roll_rad])
+        return desired_rot.as_matrix()
+    
+    def set_target_pose(self, pose_array):
+        """Set target pose from input array."""
+        self.target_pos = np.array(pose_array[:3])
+        if len(pose_array) == 6:
+            self.target_rpy = list(pose_array[3:6])
+        else:
+            self.target_rpy = [180.0, 0.0, 90.0] # Default orientation if not provided
+    
+    def update_jacobian(self, jacobian_flat, shape):
+        """Update current jacobian and initialize DOF if needed."""
+        jacobian_dof = shape[1]
+        self.current_jacobian = jacobian_flat.reshape(shape)
+
+        if self.dof is None:
+            if self.current_joint_pos is not None:
+                self._initialize_robot(self.current_joint_pos, jacobian_dof)
+            else:
+                self.dof = jacobian_dof
+        elif self.dof != jacobian_dof:
+            self.dof = jacobian_dof
+            self.home_pos = np.zeros(self.dof)
+    
+    def apply_differential_ik_control(self):
+        """Apply differential IK control with nullspace projection."""
+        if self.current_ee_pose is None or self.current_jacobian is None:
+            return self.current_joint_pos
+            
+        current_ee_pos = self.current_ee_pose['position']
+        current_ee_rpy = self.current_ee_pose['rpy']
+
+        # Calculate position and orientation error
+        pos_error = self.target_pos - current_ee_pos
+        twist = np.zeros(6)
+        twist[:3] = self.Kpos * pos_error / self.integration_dt
+        
+        # Convert current RPY to rotation matrix
+        current_rot = Rotation.from_euler('XYZ', current_ee_rpy)
+        desired_rot = Rotation.from_matrix(self.get_target_rotation_matrix())
+        rot_error = (desired_rot * current_rot.inv()).as_rotvec()
+        twist[3:] = self.Kori * rot_error / self.integration_dt
+        jac = self.current_jacobian
+        
+        # Damped least squares inverse kinematics
+        diag = self.damping * np.eye(6)
+        dq = jac.T @ np.linalg.solve(jac @ jac.T + diag, twist)
+        
+        # # Apply nullspace control
+        # # Calculate nullspace projection # uncomment to enable nullspace control
+        
+        # current_arm = self.current_joint_pos[:self.dof]
+        # jac_pinv = np.linalg.pinv(jac)
+        # N = np.eye(self.dof) - jac_pinv @ jac
+        
+        # # Apply gains to pull towards home position
+        # k_null = np.ones(self.dof) * 5.0
+        # dq_null = k_null * (self.home_pos - current_arm)  # Nullspace velocity
+        # dq += N @ dq_null  # Nullspace movement
+        
+        # Limit joint velocities
+        dq_abs_max = np.abs(dq).max()
+        if dq_abs_max > self.max_angvel:
+            dq *= self.max_angvel / dq_abs_max
+            
+        # Create full joint command (apply IK result to arm joints, keep other joints unchanged)
+        new_joints = self.current_joint_pos.copy()
+        new_joints[:self.dof] += dq * self.integration_dt
+        
+        return new_joints
+
+def main():
+    node = Node()
+    controller = Controller()
+    
+    for event in node:
+        if event["type"] == "INPUT":
+            if event["id"] == "joint_positions":
+                joint_pos = event["value"].to_numpy()
+
+                if controller.current_joint_pos is None:
+                    controller._initialize_robot(joint_pos)
+                else:
+                    controller.current_joint_pos = joint_pos
+                
+                # Request FK computation
+                node.send_output(
+                    "fk_request", 
+                    pa.array(controller.current_joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jointstate", "timestamp": time.time()}
+                )
+                
+                # Request Jacobian computation
+                node.send_output(
+                    "jacobian_request", 
+                    pa.array(controller.current_joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jacobian", "timestamp": time.time()}
+                )
+                joint_commands = controller.apply_differential_ik_control()
+                
+                # Send control commands to the robot
+                node.send_output(
+                    "joint_commands", 
+                    pa.array(joint_commands, type=pa.float32()),
+                    metadata={"timestamp": time.time()}
+                )
+
+            # Handle target pose updates
+            if event["id"] == "target_pose":
+                target_pose = event["value"].to_numpy()
+                controller.set_target_pose(target_pose)
+            
+            # Handle FK results
+            if event["id"] == "fk_result":
+                if event["metadata"]["encoding"] == "xyzrpy":
+                    ee_pose = event["value"].to_numpy()
+                    controller.current_ee_pose = {'position': ee_pose[:3],'rpy': ee_pose[3:6]}
+            
+            # Handle Jacobian results
+            if event["id"] == "jacobian_result":
+                if event["metadata"]["encoding"] == "jacobian_result":
+                    jacobian_flat = event["value"].to_numpy()
+                    jacobian_shape = event["metadata"]["jacobian_shape"]
+                    controller.update_jacobian(jacobian_flat, jacobian_shape)
+
+if __name__ == "__main__":
+    main()

examples/mujoco-sim/target_pose_control/nodes/controller_ik.py

@@ -0,0 +1,89 @@
+import numpy as np
+import time
+import pyarrow as pa
+from dora import Node
+
+class Controller:
+    def __init__(self):
+        """
+        Initialize the simple controller
+        """
+        # State variables
+        self.dof = None 
+        self.current_joint_pos = None
+        
+        self.target_pos = np.array([0.4, 0.0, 0.3])  # Conservative default target
+        self.target_rpy = [180.0, 0.0, 90.0]  # Downward orientation
+        self.ik_request_sent = True 
+        
+        print("Simple Controller initialized")
+    
+    def _initialize_robot(self, positions):
+        """Initialize controller state with appropriate dimensions"""
+        self.full_joint_count = len(positions)
+        self.current_joint_pos = positions.copy()
+        if self.dof is None:
+            self.dof = self.full_joint_count
+    
+    def set_target_pose(self, pose_array):
+        """Set target pose from input array."""
+        self.target_pos = np.array(pose_array[:3])
+        if len(pose_array) == 6:
+            self.target_rpy = list(pose_array[3:6])
+        else:
+            self.target_rpy = [180.0, 0.0, 90.0] # Default orientation if not provided
+        self.ik_request_sent = False
+    
+    def get_target_pose_array(self):
+        """Get target pose as 6D array [x, y, z, roll, pitch, yaw]"""
+        return np.concatenate([self.target_pos, self.target_rpy])
+
+def main():
+    node = Node()
+    controller = Controller()
+    
+    for event in node:
+        if event["type"] == "INPUT":
+            if event["id"] == "joint_positions":
+                joint_pos = event["value"].to_numpy()
+
+                if controller.current_joint_pos is None:
+                    controller._initialize_robot(joint_pos)
+                else:
+                    controller.current_joint_pos = joint_pos
+                
+                # Request IK solution directly
+                target_pose = controller.get_target_pose_array()
+                if not controller.ik_request_sent:
+                    node.send_output(
+                        "ik_request", 
+                        pa.array(target_pose, type=pa.float32()),
+                        metadata={"encoding": "xyzrpy", "timestamp": time.time()}
+                    )
+                    controller.ik_request_sent = True
+                
+                node.send_output(
+                    "joint_state", 
+                    pa.array(joint_pos, type=pa.float32()),
+                    metadata={"encoding": "jointstate", "timestamp": time.time()}
+                )
+
+            # Handle target pose updates
+            if event["id"] == "target_pose":
+                target_pose = event["value"].to_numpy()
+                controller.set_target_pose(target_pose)
+            
+            # Handle IK results and send directly as joint commands
+            if event["id"] == "ik_request":
+                if event["metadata"]["encoding"] == "jointstate":
+                    ik_solution = event["value"].to_numpy()
+                    # print("ik_solution", ik_solution)
+                    # Send IK solution directly as joint commands
+                    node.send_output(
+                        "joint_commands", 
+                        pa.array(ik_solution, type=pa.float32()),
+                        metadata={"timestamp": time.time()}
+                    )
+
+if __name__ == "__main__":
+    main()

examples/mujoco-sim/target_pose_control/nodes/pose_publisher.py

@@ -0,0 +1,44 @@
+import time
+import pyarrow as pa
+from dora import Node
+
+class PosePublisher:
+    """Publishes target poses in sequence."""
+    
+    def __init__(self):
+        """Initialize pose publisher with predefined target poses sequence."""
+        # target poses [x, y, z, roll, pitch, yaw]
+        self.target_poses = [
+            [0.5, 0.5, 0.3, 180.0, 0.0, 90.0],   
+            [0.6, 0.2, 0.5, 180.0, 0.0, 45.0],  
+            [0.7, 0.1, 0.4, 90.0, 90.0, 90.0], 
+            [0.4, -0.3, 0.5, 180.0, 0.0, 135.0],   
+            [-0.3, -0.6, 0.3, 180.0, 0.0, 90.0],   
+        ]
+        
+        self.current_pose_index = 0
+        print("Pose Publisher initialized")
+    
+    def get_next_pose(self):
+        """Get the next target pose in sequence."""
+        pose = self.target_poses[self.current_pose_index]
+        self.current_pose_index = (self.current_pose_index + 1) % len(self.target_poses)
+        return pose
+
+def main():
+    node = Node("pose_publisher")
+    publisher = PosePublisher()
+    time.sleep(3)  # Allow time for simulation to start
+    for event in node:
+        if event["type"] == "INPUT" and event["id"] == "tick":
+            target_pose = publisher.get_next_pose()
+            print(f"Publishing target pose: {target_pose}")
+
+            node.send_output(
+                "target_pose",
+                pa.array(target_pose, type=pa.float64()),
+                metadata={"timestamp": time.time()}
+            )
+
+if __name__ == "__main__":
+    main()

examples/multiple-daemons/node/src/main.rs

@@ -26,7 +26,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => println!("Received manual stop"),
+            Event::Stop(_) => println!("Received stop"),
             other => eprintln!("Received unexpected input: {other:?}"),
         }
     }

examples/multiple-daemons/run.rs

@@ -1,3 +1,4 @@
+use dora_cli::session::DataflowSession;
 use dora_coordinator::{ControlEvent, Event};
 use dora_core::{
     descriptor::{read_as_descriptor, DescriptorExt},
@@ -8,7 +9,7 @@ use dora_message::{
     common::DaemonId,
     coordinator_to_cli::{ControlRequestReply, DataflowIdAndName},
 };
-use dora_tracing::set_up_tracing;
+use dora_tracing::TracingBuilder;
 use eyre::{bail, Context};
 
 use std::{
@@ -29,7 +30,9 @@ use uuid::Uuid;
 
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
-    set_up_tracing("multiple-daemon-runner").wrap_err("failed to set up tracing subscriber")?;
+    TracingBuilder::new("multiple-daemon-runner")
+        .with_stdout("debug")
+        .build()?;
 
     let root = Path::new(env!("CARGO_MANIFEST_DIR"));
     std::env::set_current_dir(root.join(file!()).parent().unwrap())
@@ -47,12 +50,15 @@ async fn main() -> eyre::Result<()> {
         IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
         DORA_COORDINATOR_PORT_CONTROL_DEFAULT,
     );
-    let (_coordinator_port, coordinator) = dora_coordinator::start(
+    let (coordinator_port, coordinator) = dora_coordinator::start(
         coordinator_bind,
         coordinator_control_bind,
         ReceiverStream::new(coordinator_events_rx),
     )
     .await?;
+
+    tracing::info!("coordinator running on {coordinator_port}");
+
     let coordinator_addr = Ipv4Addr::LOCALHOST;
     let daemon_a = run_daemon(coordinator_addr.to_string(), "A");
     let daemon_b = run_daemon(coordinator_addr.to_string(), "B");
@@ -135,12 +141,17 @@ async fn start_dataflow(
         .check(&working_dir)
         .wrap_err("could not validate yaml")?;
 
+    let dataflow_session =
+        DataflowSession::read_session(dataflow).context("failed to read DataflowSession")?;
+
     let (reply_sender, reply) = oneshot::channel();
     coordinator_events_tx
         .send(Event::Control(ControlEvent::IncomingRequest {
             request: ControlRequest::Start {
+                build_id: dataflow_session.build_id,
+                session_id: dataflow_session.session_id,
                 dataflow: dataflow_descriptor,
-                local_working_dir: working_dir,
+                local_working_dir: Some(working_dir),
                 name: None,
                 uv: false,
             },
@@ -149,7 +160,21 @@ async fn start_dataflow(
         .await?;
     let result = reply.await??;
     let uuid = match result {
-        ControlRequestReply::DataflowStarted { uuid } => uuid,
+        ControlRequestReply::DataflowStartTriggered { uuid } => uuid,
+        ControlRequestReply::Error(err) => bail!("{err}"),
+        other => bail!("unexpected start dataflow reply: {other:?}"),
+    };
+
+    let (reply_sender, reply) = oneshot::channel();
+    coordinator_events_tx
+        .send(Event::Control(ControlEvent::IncomingRequest {
+            request: ControlRequest::WaitForSpawn { dataflow_id: uuid },
+            reply_sender,
+        }))
+        .await?;
+    let result = reply.await??;
+    let uuid = match result {
+        ControlRequestReply::DataflowSpawned { uuid } => uuid,
         ControlRequestReply::Error(err) => bail!("{err}"),
         other => bail!("unexpected start dataflow reply: {other:?}"),
     };
@@ -215,6 +240,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -227,6 +253,7 @@ async fn run_daemon(coordinator: String, machine_id: &str) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--machine-id")

examples/multiple-daemons/sink/src/main.rs

@@ -24,8 +24,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/python-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-multi-env/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-operator-dataflow/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/python-ros2-dataflow/run.rs

@@ -44,6 +44,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -52,6 +53,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/rerun-viewer/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");

examples/rust-dataflow-git/.gitignore

@@ -0,0 +1,4 @@
+/build
+/git
+
+/dataflow.dora-session.yaml

examples/rust-dataflow-git/README.md

@@ -0,0 +1,7 @@
+# Git-based Rust example
+
+To get started:
+
+```bash
+cargo run --example rust-dataflow-git
+```

examples/rust-dataflow-git/dataflow.yml

@@ -0,0 +1,29 @@
+nodes:
+  - id: rust-node
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-node
+    path: target/debug/rust-dataflow-example-node
+    inputs:
+      tick: dora/timer/millis/10
+    outputs:
+      - random
+
+  - id: rust-status-node
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-status-node
+    path: target/debug/rust-dataflow-example-status-node
+    inputs:
+      tick: dora/timer/millis/100
+      random: rust-node/random
+    outputs:
+      - status
+
+  - id: rust-sink
+    git: https://github.com/dora-rs/dora.git
+    rev: 64ab0d7c # pinned commit, update this when changing the message crate
+    build: cargo build -p rust-dataflow-example-sink
+    path: target/debug/rust-dataflow-example-sink
+    inputs:
+      message: rust-status-node/status

examples/rust-dataflow-git/run.rs

@@ -0,0 +1,53 @@
+use dora_tracing::set_up_tracing;
+use eyre::{bail, Context};
+use std::path::Path;
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+    set_up_tracing("rust-dataflow-runner").wrap_err("failed to set up tracing subscriber")?;
+
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    std::env::set_current_dir(root.join(file!()).parent().unwrap())
+        .wrap_err("failed to set working dir")?;
+
+    let args: Vec<String> = std::env::args().collect();
+    let dataflow = if args.len() > 1 {
+        Path::new(&args[1])
+    } else {
+        Path::new("dataflow.yml")
+    };
+    build_dataflow(dataflow).await?;
+
+    run_dataflow(dataflow).await?;
+
+    Ok(())
+}
+
+async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO").unwrap();
+    let mut cmd = tokio::process::Command::new(&cargo);
+    cmd.arg("run");
+    cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
+    cmd.arg("--").arg("build").arg(dataflow);
+    if !cmd.status().await?.success() {
+        bail!("failed to build dataflow");
+    };
+    Ok(())
+}
+
+async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO").unwrap();
+    let mut cmd = tokio::process::Command::new(&cargo);
+    cmd.arg("run");
+    cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
+    cmd.arg("--")
+        .arg("daemon")
+        .arg("--run-dataflow")
+        .arg(dataflow);
+    if !cmd.status().await?.success() {
+        bail!("failed to run dataflow");
+    };
+    Ok(())
+}

examples/rust-dataflow-url/.gitignore

@@ -0,0 +1 @@
+/build

examples/rust-dataflow-url/run.rs

@@ -23,6 +23,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -35,6 +36,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/rust-dataflow/node/src/main.rs

@@ -26,7 +26,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => println!("Received manual stop"),
+            Event::Stop(_) => println!("Received stop"),
             other => eprintln!("Received unexpected input: {other:?}"),
         }
     }

examples/rust-dataflow/run.rs

@@ -16,7 +16,6 @@ async fn main() -> eyre::Result<()> {
     } else {
         Path::new("dataflow.yml")
     };
-
     build_dataflow(dataflow).await?;
 
     run_dataflow(dataflow).await?;
@@ -29,6 +28,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -41,6 +41,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/rust-dataflow/sink-dynamic/src/main.rs

@@ -25,8 +25,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/rust-dataflow/sink/src/main.rs

@@ -24,8 +24,8 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("Ignoring unexpected input `{other}`"),
             },
-            Event::Stop => {
-                println!("Received manual stop");
+            Event::Stop(_) => {
+                println!("Received stop");
             }
             Event::InputClosed { id } => {
                 println!("Input `{id}` was closed");

examples/rust-dataflow/status-node/src/main.rs

@@ -29,7 +29,7 @@ fn main() -> eyre::Result<()> {
                 }
                 other => eprintln!("ignoring unexpected input {other}"),
             },
-            Event::Stop => {}
+            Event::Stop(_) => {}
             Event::InputClosed { id } => {
                 println!("input `{id}` was closed");
                 if *id == "random" {

examples/rust-ros2-dataflow/node/src/main.rs

@@ -119,7 +119,7 @@ fn main() -> eyre::Result<()> {
                     }
                     other => eprintln!("Ignoring unexpected input `{other}`"),
                 },
-                Event::Stop => println!("Received manual stop"),
+                Event::Stop(_) => println!("Received stop"),
                 other => eprintln!("Received unexpected input: {other:?}"),
             },
             MergedEvent::External(pose) => {

examples/rust-ros2-dataflow/run.rs

@@ -23,6 +23,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow);
     if !cmd.status().await?.success() {
         bail!("failed to build dataflow");
@@ -35,6 +36,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--")
         .arg("daemon")
         .arg("--run-dataflow")

examples/vggt/depth.dora-session.yaml

@@ -0,0 +1,8 @@
+build_id: 2b402c1e-e52e-45e9-86e5-236b33a77369
+session_id: 275de19c-e605-4865-bc5f-2f15916bade9
+git_sources: {}
+local_build:
+  node_working_dirs:
+    camera: /Users/xaviertao/Documents/work/dora/examples/vggt
+    dora-vggt: /Users/xaviertao/Documents/work/dora/examples/vggt
+    plot: /Users/xaviertao/Documents/work/dora/examples/vggt

examples/vggt/depth.yaml

@@ -0,0 +1,26 @@
+nodes:
+  - id: camera
+    build: pip install opencv-video-capture
+    path: opencv-video-capture
+    inputs:
+      tick: dora/timer/millis/100
+    outputs:
+      - image
+    env:
+      CAPTURE_PATH: 1
+
+  - id: dora-vggt
+    build: pip install -e ../../node-hub/dora-vggt
+    path: dora-vggt
+    inputs:
+      image: camera/image
+    outputs:
+      - depth
+      - image
+
+  - id: plot
+    build: pip install dora-rerun
+    path: dora-rerun
+    inputs:
+      camera/image: dora-vggt/image
+      camera/depth: dora-vggt/depth

examples/vggt/vggt-v-realsense.yaml

@@ -0,0 +1,29 @@
+nodes:
+  - id: camera
+    build: pip install -e ../../node-hub/dora-pyrealsense
+    path: dora-pyrealsense
+    inputs:
+      tick: dora/timer/millis/100
+    outputs:
+      - image
+      - depth
+    env:
+      CAPTURE_PATH: 8
+
+  - id: dora-vggt
+    build: pip install -e ../../node-hub/dora-vggt
+    path: dora-vggt
+    inputs:
+      image: camera/image
+    outputs:
+      - depth
+      - image
+
+  - id: plot
+    build: pip install dora-rerun
+    path: dora-rerun
+    inputs:
+      camera/image: dora-vggt/image
+      camera/depth: dora-vggt/depth
+      realsense/image: camera/image
+      realsense/depth: camera/depth

examples/vlm/run.rs

@@ -43,6 +43,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("build").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");
@@ -51,6 +52,7 @@ async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> {
     let mut cmd = tokio::process::Command::new(&cargo);
     cmd.arg("run");
     cmd.arg("--package").arg("dora-cli");
+    cmd.arg("--release");
     cmd.arg("--").arg("run").arg(dataflow).arg("--uv");
     if !cmd.status().await?.success() {
         bail!("failed to run dataflow");