Merge pull request #566 from dora-rs/v0.3.5

V0.3.5

.github/workflows/pip-release.yml

@@ -12,6 +12,7 @@ on:
   release:
     types:
       - "published"
+  workflow_dispatch:
 
 permissions:
   contents: write
@@ -38,6 +39,14 @@ jobs:
           sccache: "true"
           manylinux: auto
           working-directory: apis/python/node
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --zig
+          sccache: "true"
+          working-directory: apis/python/node
+          manylinux: musllinux_1_2
       - name: Upload wheels
         uses: actions/upload-artifact@v3
         with:

.github/workflows/release.yml

@@ -7,6 +7,7 @@ on:
   release:
     types:
       - "published"
+  workflow_dispatch:
 
 jobs:
   cargo-release:
@@ -22,6 +23,22 @@ jobs:
       - uses: actions/checkout@v3
 
       - uses: r7kamura/rust-problem-matchers@v1.1.0
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        if: runner.os == 'Linux'
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
 
       - name: "Publish packages on `crates.io`"
         if: runner.os == 'Linux'

Cargo.lock

@@ -1083,7 +1083,7 @@ dependencies = [
 
 [[package]]
 name = "benchmark-example-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -1096,7 +1096,7 @@ dependencies = [
 
 [[package]]
 name = "benchmark-example-sink"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -1744,7 +1744,7 @@ dependencies = [
 
 [[package]]
 name = "communication-layer-pub-sub"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "flume 0.10.14",
  "zenoh",
@@ -1752,7 +1752,7 @@ dependencies = [
 
 [[package]]
 name = "communication-layer-request-reply"
-version = "0.3.4"
+version = "0.3.5"
 
 [[package]]
 name = "concurrent-queue"
@@ -2252,7 +2252,7 @@ dependencies = [
 
 [[package]]
 name = "dora-arrow-convert"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "arrow",
  "eyre",
@@ -2260,7 +2260,7 @@ dependencies = [
 
 [[package]]
 name = "dora-cli"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "bat",
  "clap 4.5.8",
@@ -2295,7 +2295,7 @@ dependencies = [
 
 [[package]]
 name = "dora-coordinator"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "ctrlc",
  "dora-core",
@@ -2314,7 +2314,7 @@ dependencies = [
 
 [[package]]
 name = "dora-core"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "aligned-vec",
  "dora-message",
@@ -2334,7 +2334,7 @@ dependencies = [
 
 [[package]]
 name = "dora-daemon"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "aligned-vec",
  "async-trait",
@@ -2365,7 +2365,7 @@ dependencies = [
 
 [[package]]
 name = "dora-download"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "eyre",
  "reqwest",
@@ -2393,7 +2393,7 @@ dependencies = [
 
 [[package]]
 name = "dora-message"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "arrow-data",
  "arrow-schema",
@@ -2404,7 +2404,7 @@ dependencies = [
 
 [[package]]
 name = "dora-metrics"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "eyre",
  "opentelemetry 0.22.0",
@@ -2415,7 +2415,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "aligned-vec",
  "arrow",
@@ -2438,7 +2438,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-c"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "arrow-array",
  "dora-node-api",
@@ -2448,7 +2448,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-cxx"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -2465,7 +2465,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-python"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "arrow",
  "dora-node-api",
@@ -2484,7 +2484,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-arrow-convert",
  "dora-operator-api-macros",
@@ -2493,14 +2493,14 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-c"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-operator-api-types",
 ]
 
 [[package]]
 name = "dora-operator-api-cxx"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -2509,7 +2509,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-macros"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2518,7 +2518,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-python"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "aligned-vec",
  "arrow",
@@ -2532,7 +2532,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-types"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "arrow",
  "dora-arrow-convert",
@@ -2541,7 +2541,7 @@ dependencies = [
 
 [[package]]
 name = "dora-record"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "chrono",
  "dora-node-api",
@@ -2553,7 +2553,7 @@ dependencies = [
 
 [[package]]
 name = "dora-rerun"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -2619,7 +2619,7 @@ dependencies = [
 
 [[package]]
 name = "dora-runtime"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "aligned-vec",
  "arrow",
@@ -2646,7 +2646,7 @@ dependencies = [
 
 [[package]]
 name = "dora-tracing"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "eyre",
  "opentelemetry 0.18.0",
@@ -4968,7 +4968,7 @@ dependencies = [
 
 [[package]]
 name = "multiple-daemons-example-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -4979,14 +4979,14 @@ dependencies = [
 
 [[package]]
 name = "multiple-daemons-example-operator"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-operator-api",
 ]
 
 [[package]]
 name = "multiple-daemons-example-sink"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -8184,7 +8184,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -8195,7 +8195,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-sink"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -8203,7 +8203,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-sink-dynamic"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -8211,7 +8211,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-status-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -8230,7 +8230,7 @@ dependencies = [
 
 [[package]]
 name = "rust-ros2-dataflow-example-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "dora-node-api",
  "dora-ros2-bridge",
@@ -8742,7 +8742,7 @@ dependencies = [
 
 [[package]]
 name = "shared-memory-server"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "bincode",
  "eyre",

Cargo.toml

@@ -39,31 +39,31 @@ members = [
 
 [workspace.package]
 # Make sure to also bump `apis/node/python/__init__.py` version.
-version = "0.3.4"
+version = "0.3.5"
 description = "`dora` goal is to be a low latency, composable, and distributed data flow."
 documentation = "https://dora.carsmos.ai"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-dora-node-api = { version = "0.3.4", path = "apis/rust/node", default-features = false }
-dora-node-api-python = { version = "0.3.4", path = "apis/python/node", default-features = false }
-dora-operator-api = { version = "0.3.4", path = "apis/rust/operator", default-features = false }
-dora-operator-api-macros = { version = "0.3.4", path = "apis/rust/operator/macros" }
-dora-operator-api-types = { version = "0.3.4", path = "apis/rust/operator/types" }
-dora-operator-api-python = { version = "0.3.4", path = "apis/python/operator" }
-dora-operator-api-c = { version = "0.3.4", path = "apis/c/operator" }
-dora-node-api-c = { version = "0.3.4", path = "apis/c/node" }
-dora-core = { version = "0.3.4", path = "libraries/core" }
-dora-arrow-convert = { version = "0.3.4", path = "libraries/arrow-convert" }
-dora-tracing = { version = "0.3.4", path = "libraries/extensions/telemetry/tracing" }
-dora-metrics = { version = "0.3.4", path = "libraries/extensions/telemetry/metrics" }
-dora-download = { version = "0.3.4", path = "libraries/extensions/download" }
-shared-memory-server = { version = "0.3.4", path = "libraries/shared-memory-server" }
-communication-layer-request-reply = { version = "0.3.4", path = "libraries/communication-layer/request-reply" }
-dora-message = { version = "0.3.4", path = "libraries/message" }
-dora-runtime = { version = "0.3.4", path = "binaries/runtime" }
-dora-daemon = { version = "0.3.4", path = "binaries/daemon" }
-dora-coordinator = { version = "0.3.4", path = "binaries/coordinator" }
+dora-node-api = { version = "0.3.5", path = "apis/rust/node", default-features = false }
+dora-node-api-python = { version = "0.3.5", path = "apis/python/node", default-features = false }
+dora-operator-api = { version = "0.3.5", path = "apis/rust/operator", default-features = false }
+dora-operator-api-macros = { version = "0.3.5", path = "apis/rust/operator/macros" }
+dora-operator-api-types = { version = "0.3.5", path = "apis/rust/operator/types" }
+dora-operator-api-python = { version = "0.3.5", path = "apis/python/operator" }
+dora-operator-api-c = { version = "0.3.5", path = "apis/c/operator" }
+dora-node-api-c = { version = "0.3.5", path = "apis/c/node" }
+dora-core = { version = "0.3.5", path = "libraries/core" }
+dora-arrow-convert = { version = "0.3.5", path = "libraries/arrow-convert" }
+dora-tracing = { version = "0.3.5", path = "libraries/extensions/telemetry/tracing" }
+dora-metrics = { version = "0.3.5", path = "libraries/extensions/telemetry/metrics" }
+dora-download = { version = "0.3.5", path = "libraries/extensions/download" }
+shared-memory-server = { version = "0.3.5", path = "libraries/shared-memory-server" }
+communication-layer-request-reply = { version = "0.3.5", path = "libraries/communication-layer/request-reply" }
+dora-message = { version = "0.3.5", path = "libraries/message" }
+dora-runtime = { version = "0.3.5", path = "binaries/runtime" }
+dora-daemon = { version = "0.3.5", path = "binaries/daemon" }
+dora-coordinator = { version = "0.3.5", path = "binaries/coordinator" }
 dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" }
 dora-ros2-bridge-msg-gen = { path = "libraries/extensions/ros2-bridge/msg-gen" }
 dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" }

Changelog.md

@@ -1,5 +1,54 @@
 # Changelog
 
+## v0.3.5 (2024-07-03)
+
+## What's Changed
+
+- chore: Support RISCV64 by @LyonRust in https://github.com/dora-rs/dora/pull/505
+- Json schemas for VSCode YAML Support by @haixuanTao in https://github.com/dora-rs/dora/pull/497
+- Pretty Print Rust object when called from Python print by @haixuanTao in https://github.com/dora-rs/dora/pull/503
+- Fix `Cargo.lock` by @phil-opp in https://github.com/dora-rs/dora/pull/506
+- Use dependabot for automatic lockfile updates by @phil-opp in https://github.com/dora-rs/dora/pull/507
+- Run cargo update by @phil-opp in https://github.com/dora-rs/dora/pull/508
+- Allow top-level fields in node declaration by @phil-opp in https://github.com/dora-rs/dora/pull/478
+- Configure Renovate by @renovate in https://github.com/dora-rs/dora/pull/509
+- Make non-UTF8 stdout/stderr from nodes non-fatal by @phil-opp in https://github.com/dora-rs/dora/pull/510
+- Make dora cli connect to remote coordinator by @Gege-Wang in https://github.com/dora-rs/dora/pull/513
+- Provide help messages for CLI by @phil-opp in https://github.com/dora-rs/dora/pull/519
+- Renovate: group all dependency updates in single PR by @phil-opp in https://github.com/dora-rs/dora/pull/524
+- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/529
+- Improve coordinator port config by @phil-opp in https://github.com/dora-rs/dora/pull/520
+- Fix some typos and add automatic typos check to CI by @EricLBuehler in https://github.com/dora-rs/dora/pull/539
+- Update Pyo3 bounds by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/472
+- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/543
+- Small logging improvements by @phil-opp in https://github.com/dora-rs/dora/pull/537
+- Refuse relative path for remote in coordinator by @XxChang in https://github.com/dora-rs/dora/pull/538
+- chore(deps): update rust crate clap to v4.5.7 by @renovate in https://github.com/dora-rs/dora/pull/546
+- Add `--quiet` flag to daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/548
+- Implement file-based logging in daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/549
+- Spawn daemon and coordinator in quiet mode on `dora up` by @phil-opp in https://github.com/dora-rs/dora/pull/550
+- Run dynamic node by @haixuanTao in https://github.com/dora-rs/dora/pull/517
+- Update dora new by @XxChang in https://github.com/dora-rs/dora/pull/553
+- fix event_as_input bug by @XxChang in https://github.com/dora-rs/dora/pull/556
+- Transform custom PyEvent into standard python dictionary for easier d… by @haixuanTao in https://github.com/dora-rs/dora/pull/557
+- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/558
+- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/560
+- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/563
+- Print only first node error and report more metadata in dataflow results by @phil-opp in https://github.com/dora-rs/dora/pull/552
+- Make `dora start` attach by default, add `--detach` to opt-out by @phil-opp in https://github.com/dora-rs/dora/pull/561
+- List failed and finished dataflows in `dora list` by @phil-opp in https://github.com/dora-rs/dora/pull/554
+- Ignore-quicker-pending-drop-token by @haixuanTao in https://github.com/dora-rs/dora/pull/568
+- Increasing grace duration to 2 seconds so that drop token get well returned in https://github.com/dora-rs/dora/pull/576
+
+## New Contributors
+
+- @LyonRust made their first contribution in https://github.com/dora-rs/dora/pull/505
+- @renovate made their first contribution in https://github.com/dora-rs/dora/pull/509
+- @Gege-Wang made their first contribution in https://github.com/dora-rs/dora/pull/513
+- @EricLBuehler made their first contribution in https://github.com/dora-rs/dora/pull/539
+
+**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.4...v0.3.5
+
 ## v0.3.4 (2024-05-17)
 
 ## What's Changed

README.md

@@ -160,17 +160,17 @@ For more info on installation, check out [our guide](https://www.dora-rs.ai/docs
 1. Install the example python dependencies:
 
 ```bash
-pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/requirements.txt
+pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/requirements.txt
 ```
 
 2. Get some example operators:
 
 ```bash
-wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/webcam.py
-wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/plot.py
-wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/utils.py
-wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/object_detection.py
-wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/dataflow.yml
+wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/webcam.py
+wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/plot.py
+wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/utils.py
+wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/object_detection.py
+wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/dataflow.yml
 ```
 
 3. Start the dataflow

apis/rust/node/src/event_stream/thread.rs

@@ -220,14 +220,14 @@ fn report_remaining_drop_tokens(
 
         let mut still_pending = Vec::new();
         for (token, rx, since, _) in pending_drop_tokens.drain(..) {
-            match rx.recv_timeout(Duration::from_millis(100)) {
+            match rx.recv_timeout(Duration::from_millis(50)) {
                 Ok(()) => return Err(eyre!("Node API should not send anything on ACK channel")),
                 Err(flume::RecvTimeoutError::Disconnected) => {
                     // the event was dropped -> add the drop token to the list
                     drop_tokens.push(token);
                 }
                 Err(flume::RecvTimeoutError::Timeout) => {
-                    let duration = Duration::from_secs(30);
+                    let duration = Duration::from_millis(200);
                     if since.elapsed() > duration {
                         tracing::warn!(
                             "timeout: node finished, but token {token:?} was still not \

apis/rust/node/src/node/mod.rs

@@ -401,7 +401,7 @@ impl Drop for DoraNode {
                 );
             }
 
-            match self.drop_stream.recv_timeout(Duration::from_secs(10)) {
+            match self.drop_stream.recv_timeout(Duration::from_millis(500)) {
                 Ok(token) => {
                     self.sent_out_shared_memory.remove(&token);
                 }