diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml index 923df826..70cbb21b 100644 --- a/.github/workflows/pip-release.yml +++ b/.github/workflows/pip-release.yml @@ -12,6 +12,7 @@ on: release: types: - "published" + workflow_dispatch: permissions: contents: write @@ -38,6 +39,14 @@ jobs: sccache: "true" manylinux: auto working-directory: apis/python/node + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --zig + sccache: "true" + working-directory: apis/python/node + manylinux: musllinux_1_2 - name: Upload wheels uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 776ae2dd..f7d747d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: release: types: - "published" + workflow_dispatch: jobs: cargo-release: @@ -22,6 +23,22 @@ jobs: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + if: runner.os == 'Linux' + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: true - name: "Publish packages on `crates.io`" if: runner.os == 'Linux' diff --git a/Cargo.lock b/Cargo.lock index 3eb5d109..50d85a67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1083,7 +1083,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -1096,7 +1096,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -1744,7 +1744,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.4" +version = "0.3.5" dependencies = [ "flume 0.10.14", "zenoh", @@ -1752,7 +1752,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.4" +version = "0.3.5" [[package]] name = "concurrent-queue" @@ -2252,7 +2252,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arrow", "eyre", @@ -2260,7 +2260,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.4" +version = "0.3.5" dependencies = [ "bat", "clap 4.5.8", @@ -2295,7 +2295,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.4" +version = "0.3.5" dependencies = [ "ctrlc", "dora-core", @@ -2314,7 +2314,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aligned-vec", "dora-message", @@ -2334,7 +2334,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aligned-vec", "async-trait", @@ -2365,7 +2365,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.4" +version = "0.3.5" dependencies = [ "eyre", "reqwest", @@ -2393,7 +2393,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arrow-data", "arrow-schema", @@ -2404,7 +2404,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.4" +version = "0.3.5" dependencies = [ "eyre", "opentelemetry 0.22.0", @@ -2415,7 +2415,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aligned-vec", "arrow", @@ -2438,7 +2438,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arrow-array", "dora-node-api", @@ -2448,7 +2448,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.4" +version = "0.3.5" dependencies = [ "cxx", "cxx-build", @@ -2465,7 +2465,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arrow", "dora-node-api", @@ -2484,7 +2484,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -2493,14 +2493,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.4" +version = "0.3.5" dependencies = [ "cxx", "cxx-build", @@ -2509,7 +2509,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.4" +version = "0.3.5" dependencies = [ "proc-macro2", "quote", @@ -2518,7 +2518,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aligned-vec", "arrow", @@ -2532,7 +2532,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arrow", "dora-arrow-convert", @@ -2541,7 +2541,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.4" +version = "0.3.5" dependencies = [ "chrono", "dora-node-api", @@ -2553,7 +2553,7 @@ dependencies = [ [[package]] name = "dora-rerun" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -2619,7 +2619,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aligned-vec", "arrow", @@ -2646,7 +2646,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.4" +version = "0.3.5" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -4968,7 +4968,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -4979,14 +4979,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -8184,7 +8184,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -8195,7 +8195,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-sink" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -8203,7 +8203,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-sink-dynamic" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -8211,7 +8211,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-status-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "eyre", @@ -8230,7 +8230,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -8742,7 +8742,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.4" +version = "0.3.5" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 9fa48de3..798b7a2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,31 +39,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.4" +version = "0.3.5" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.4", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.4", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.4", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.4", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.4", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.4", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.4", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.4", path = "apis/c/node" } -dora-core = { version = "0.3.4", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.4", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.4", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.4", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.4", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.4", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.4", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.4", path = "libraries/message" } -dora-runtime = { version = "0.3.4", path = "binaries/runtime" } -dora-daemon = { version = "0.3.4", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.4", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.5", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.5", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.5", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.5", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.5", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.5", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.5", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.5", path = "apis/c/node" } +dora-core = { version = "0.3.5", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.5", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.5", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.5", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.5", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.5", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.5", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.5", path = "libraries/message" } +dora-runtime = { version = "0.3.5", path = "binaries/runtime" } +dora-daemon = { version = "0.3.5", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.5", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-msg-gen = { path = "libraries/extensions/ros2-bridge/msg-gen" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } diff --git a/Changelog.md b/Changelog.md index 8f0ca9cd..1c14716b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,54 @@ # Changelog +## v0.3.5 (2024-07-03) + +## What's Changed + +- chore: Support RISCV64 by @LyonRust in https://github.com/dora-rs/dora/pull/505 +- Json schemas for VSCode YAML Support by @haixuanTao in https://github.com/dora-rs/dora/pull/497 +- Pretty Print Rust object when called from Python print by @haixuanTao in https://github.com/dora-rs/dora/pull/503 +- Fix `Cargo.lock` by @phil-opp in https://github.com/dora-rs/dora/pull/506 +- Use dependabot for automatic lockfile updates by @phil-opp in https://github.com/dora-rs/dora/pull/507 +- Run cargo update by @phil-opp in https://github.com/dora-rs/dora/pull/508 +- Allow top-level fields in node declaration by @phil-opp in https://github.com/dora-rs/dora/pull/478 +- Configure Renovate by @renovate in https://github.com/dora-rs/dora/pull/509 +- Make non-UTF8 stdout/stderr from nodes non-fatal by @phil-opp in https://github.com/dora-rs/dora/pull/510 +- Make dora cli connect to remote coordinator by @Gege-Wang in https://github.com/dora-rs/dora/pull/513 +- Provide help messages for CLI by @phil-opp in https://github.com/dora-rs/dora/pull/519 +- Renovate: group all dependency updates in single PR by @phil-opp in https://github.com/dora-rs/dora/pull/524 +- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/529 +- Improve coordinator port config by @phil-opp in https://github.com/dora-rs/dora/pull/520 +- Fix some typos and add automatic typos check to CI by @EricLBuehler in https://github.com/dora-rs/dora/pull/539 +- Update Pyo3 bounds by @Michael-J-Ward in https://github.com/dora-rs/dora/pull/472 +- chore(deps): update dependencies by @renovate in https://github.com/dora-rs/dora/pull/543 +- Small logging improvements by @phil-opp in https://github.com/dora-rs/dora/pull/537 +- Refuse relative path for remote in coordinator by @XxChang in https://github.com/dora-rs/dora/pull/538 +- chore(deps): update rust crate clap to v4.5.7 by @renovate in https://github.com/dora-rs/dora/pull/546 +- Add `--quiet` flag to daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/548 +- Implement file-based logging in daemon and coordinator by @phil-opp in https://github.com/dora-rs/dora/pull/549 +- Spawn daemon and coordinator in quiet mode on `dora up` by @phil-opp in https://github.com/dora-rs/dora/pull/550 +- Run dynamic node by @haixuanTao in https://github.com/dora-rs/dora/pull/517 +- Update dora new by @XxChang in https://github.com/dora-rs/dora/pull/553 +- fix event_as_input bug by @XxChang in https://github.com/dora-rs/dora/pull/556 +- Transform custom PyEvent into standard python dictionary for easier d… by @haixuanTao in https://github.com/dora-rs/dora/pull/557 +- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/558 +- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/560 +- Update dependencies by @renovate in https://github.com/dora-rs/dora/pull/563 +- Print only first node error and report more metadata in dataflow results by @phil-opp in https://github.com/dora-rs/dora/pull/552 +- Make `dora start` attach by default, add `--detach` to opt-out by @phil-opp in https://github.com/dora-rs/dora/pull/561 +- List failed and finished dataflows in `dora list` by @phil-opp in https://github.com/dora-rs/dora/pull/554 +- Ignore-quicker-pending-drop-token by @haixuanTao in https://github.com/dora-rs/dora/pull/568 +- Increasing grace duration to 2 seconds so that drop token get well returned in https://github.com/dora-rs/dora/pull/576 + +## New Contributors + +- @LyonRust made their first contribution in https://github.com/dora-rs/dora/pull/505 +- @renovate made their first contribution in https://github.com/dora-rs/dora/pull/509 +- @Gege-Wang made their first contribution in https://github.com/dora-rs/dora/pull/513 +- @EricLBuehler made their first contribution in https://github.com/dora-rs/dora/pull/539 + +**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.4...v0.3.5 + ## v0.3.4 (2024-05-17) ## What's Changed diff --git a/README.md b/README.md index 1a6909da..12853b42 100644 --- a/README.md +++ b/README.md @@ -160,17 +160,17 @@ For more info on installation, check out [our guide](https://www.dora-rs.ai/docs 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.4/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.5/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/rust/node/src/event_stream/thread.rs b/apis/rust/node/src/event_stream/thread.rs index bee8cc22..d34e2ab8 100644 --- a/apis/rust/node/src/event_stream/thread.rs +++ b/apis/rust/node/src/event_stream/thread.rs @@ -220,14 +220,14 @@ fn report_remaining_drop_tokens( let mut still_pending = Vec::new(); for (token, rx, since, _) in pending_drop_tokens.drain(..) { - match rx.recv_timeout(Duration::from_millis(100)) { + match rx.recv_timeout(Duration::from_millis(50)) { Ok(()) => return Err(eyre!("Node API should not send anything on ACK channel")), Err(flume::RecvTimeoutError::Disconnected) => { // the event was dropped -> add the drop token to the list drop_tokens.push(token); } Err(flume::RecvTimeoutError::Timeout) => { - let duration = Duration::from_secs(30); + let duration = Duration::from_millis(200); if since.elapsed() > duration { tracing::warn!( "timeout: node finished, but token {token:?} was still not \ diff --git a/apis/rust/node/src/node/mod.rs b/apis/rust/node/src/node/mod.rs index 9eb4b18e..277084d3 100644 --- a/apis/rust/node/src/node/mod.rs +++ b/apis/rust/node/src/node/mod.rs @@ -401,7 +401,7 @@ impl Drop for DoraNode { ); } - match self.drop_stream.recv_timeout(Duration::from_secs(10)) { + match self.drop_stream.recv_timeout(Duration::from_millis(500)) { Ok(token) => { self.sent_out_shared_memory.remove(&token); }