From adcfeaf3b835a509f874edc98f204ffbfe8460ae Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:28:37 +0100 Subject: [PATCH 001/104] Fix python path error using path searching for both `python3` and `python` Fix #394 --- Cargo.lock | 1 + binaries/daemon/Cargo.toml | 1 + binaries/daemon/src/spawn.rs | 7 ++++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f074414b..3ad9cc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1491,6 +1491,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "uuid", + "which", ] [[package]] diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index f580ca30..e2ca4fab 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -39,3 +39,4 @@ bincode = "1.3.3" async-trait = "0.1.64" arrow-schema = { workspace = true } aligned-vec = "0.5.0" +which = "4.3.0" diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 1a79bda0..e1e25a77 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -137,7 +137,12 @@ pub async fn spawn_node( let mut command = if has_python_operator && !has_other_operator { // Use python to spawn runtime if there is a python operator - let mut command = tokio::process::Command::new("python3"); + let python = match which::which("python3") { + Ok(python) => python, + Err(_) => which::which("python") + .context("failed to find `python` or `python3` in dora-daemon path. Make sure that python is available for the daemon.")?, + }; + let mut command = tokio::process::Command::new(python); command.args([ "-c", format!("import dora; dora.start_runtime() # {}", node.id).as_str(), From e875b3a8e8e3eaed70ca5353467e541f25f661ad Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:31:08 +0100 Subject: [PATCH 002/104] Test: Readding windows CI/CD --- .github/workflows/ci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99c2a214..505a41d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,13 +65,7 @@ jobs: - name: "Build" run: cargo build --all - name: "Test" - # Remove Windows as there is `pdb` linker issue. - # See: https://github.com/dora-rs/dora/pull/359#discussion_r1360268497 - if: runner.os == 'Linux' || runner.os == 'macOS' run: cargo test --all - - name: "Test" - if: runner.os == 'Windows' - run: cargo test --all --lib # Run examples as separate job because otherwise we will exhaust the disk # space of the GitHub action runners. @@ -138,14 +132,9 @@ jobs: # python examples - uses: actions/setup-python@v2 - if: runner.os == 'Linux' || runner.os == 'macOS' - with: - python-version: "3.8" - name: "Python Dataflow example" - if: runner.os == 'Linux' || runner.os == 'macOS' run: cargo run --example python-dataflow - name: "Python Operator Dataflow example" - if: runner.os == 'Linux' || runner.os == 'macOS' run: cargo run --example python-operator-dataflow # ROS2 bridge examples From 06356ffe256739ccc822e857f97edc806682b6a6 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:40:03 +0100 Subject: [PATCH 003/104] Renaming `dora-cli` dora --- Cargo.lock | 18 +++++++++--------- binaries/cli/Cargo.toml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f074414b..8406497b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1390,15 +1390,7 @@ dependencies = [ ] [[package]] -name = "dora-arrow-convert" -version = "0.3.0" -dependencies = [ - "arrow", - "eyre", -] - -[[package]] -name = "dora-cli" +name = "dora" version = "0.3.0" dependencies = [ "bat", @@ -1421,6 +1413,14 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "dora-arrow-convert" +version = "0.3.0" +dependencies = [ + "arrow", + "eyre", +] + [[package]] name = "dora-coordinator" version = "0.3.0" diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index 8cc9d38c..004f296c 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "dora-cli" +name = "dora" version.workspace = true edition = "2021" documentation.workspace = true @@ -9,7 +9,7 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [[bin]] -name = "dora-cli" +name = "dora" path = "src/main.rs" [features] From 565bda5d99bb82b608f6a9285407279c8ac76dfa Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:42:16 +0100 Subject: [PATCH 004/104] Rename dora-cli to dora in examples --- .github/workflows/ci.yml | 18 +++++++++--------- examples/benchmark/run.rs | 2 +- examples/multiple-daemons/run.rs | 2 +- examples/rust-dataflow-url/run.rs | 2 +- examples/rust-dataflow/run.rs | 2 +- examples/rust-ros2-dataflow/run.rs | 2 +- libraries/arrow-convert/src/from_impls.rs | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99c2a214..c0df63bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,30 +243,30 @@ jobs: # fail-fast by using bash shell explictly shell: bash run: | - dora-cli up - dora-cli list + dora up + dora list # Test Rust template Project - dora-cli new test_rust_project --internal-create-with-path-dependencies + dora new test_rust_project --internal-create-with-path-dependencies cd test_rust_project cargo build --all - dora-cli start dataflow.yml --name ci-rust-test + dora start dataflow.yml --name ci-rust-test sleep 10 - dora-cli stop --name ci-rust-test + dora stop --name ci-rust-test cd .. # Test Python template Project pip3 install maturin maturin build -m apis/python/node/Cargo.toml pip3 install target/wheels/* - dora-cli new test_python_project --lang python --internal-create-with-path-dependencies + dora new test_python_project --lang python --internal-create-with-path-dependencies cd test_python_project - dora-cli start dataflow.yml --name ci-python-test + dora start dataflow.yml --name ci-python-test sleep 10 - dora-cli stop --name ci-python-test + dora stop --name ci-python-test cd .. - dora-cli destroy + dora destroy clippy: name: "Clippy" diff --git a/examples/benchmark/run.rs b/examples/benchmark/run.rs index a7d35429..834f4ebb 100644 --- a/examples/benchmark/run.rs +++ b/examples/benchmark/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora-cli"); + cmd.arg("--package").arg("dora"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index f1ee34de..f47416e3 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -204,7 +204,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora-cli"); + cmd.arg("--package").arg("dora"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-dataflow-url/run.rs b/examples/rust-dataflow-url/run.rs index dc27e137..0b25d79d 100644 --- a/examples/rust-dataflow-url/run.rs +++ b/examples/rust-dataflow-url/run.rs @@ -34,7 +34,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora-cli"); + cmd.arg("--package").arg("dora"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-dataflow/run.rs b/examples/rust-dataflow/run.rs index a7d35429..834f4ebb 100644 --- a/examples/rust-dataflow/run.rs +++ b/examples/rust-dataflow/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora-cli"); + cmd.arg("--package").arg("dora"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-ros2-dataflow/run.rs b/examples/rust-ros2-dataflow/run.rs index a7d35429..834f4ebb 100644 --- a/examples/rust-ros2-dataflow/run.rs +++ b/examples/rust-ros2-dataflow/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora-cli"); + cmd.arg("--package").arg("dora"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/libraries/arrow-convert/src/from_impls.rs b/libraries/arrow-convert/src/from_impls.rs index 8d918a3a..db484c6b 100644 --- a/libraries/arrow-convert/src/from_impls.rs +++ b/libraries/arrow-convert/src/from_impls.rs @@ -1,5 +1,5 @@ use arrow::{ - array::{make_array, Array, AsArray, PrimitiveArray, StringArray}, + array::{Array, AsArray, PrimitiveArray, StringArray}, datatypes::ArrowPrimitiveType, }; use eyre::ContextCompat; From 051f77d50ea4b4fedb1a4d41f1d6c3f291147dd3 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:42:58 +0100 Subject: [PATCH 005/104] Renaming dora-cli to dora in release CD --- .github/workflows/release.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70ad0024..148ef272 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: cargo publish -p dora-node-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Publish binaries crates - cargo publish -p dora-cli --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish -p dora --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-coordinator --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-runtime --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-daemon --token ${{ secrets.CARGO_REGISTRY_TOKEN }} @@ -83,7 +83,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora -p dora-daemon" - name: Create Archive (Windows) if: runner.os == 'Windows' @@ -92,7 +92,7 @@ jobs: New-Item -Path archive -ItemType Directory Copy-Item target/release/dora-coordinator.exe -Destination archive Copy-Item target/release/dora-daemon.exe -Destination archive - Copy-Item target/release/dora-cli.exe -Destination archive/dora.exe + Copy-Item target/release/dora.exe -Destination archive/dora.exe Compress-Archive -Path archive\* -DestinationPath archive.zip - name: "Upload release asset" @@ -122,7 +122,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora -p dora-daemon" - name: "Create Archive (Unix)" if: runner.os == 'Linux' || runner.os == 'macOS' @@ -130,7 +130,7 @@ jobs: mkdir archive cp target/release/dora-coordinator archive cp target/release/dora-daemon archive - cp target/release/dora-cli archive/dora + cp target/release/dora archive/dora cd archive zip -r ../archive.zip . cd .. @@ -166,7 +166,7 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli -p dora-daemon + args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora -p dora-daemon - name: "Archive Linux ARM64" if: runner.os == 'Linux' @@ -174,7 +174,7 @@ jobs: mkdir archive_aarch64 cp target/aarch64-unknown-linux-gnu/release/dora-coordinator archive_aarch64 cp target/aarch64-unknown-linux-gnu/release/dora-daemon archive_aarch64 - cp target/aarch64-unknown-linux-gnu/release/dora-cli archive_aarch64/dora + cp target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora cd archive_aarch64 zip -r ../archive_aarch64.zip . cd .. @@ -213,7 +213,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli -p dora-daemon + args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora -p dora-daemon - name: "Archive macOS ARM64" if: runner.os == 'macOS' @@ -221,7 +221,7 @@ jobs: mkdir archive_aarch64 cp target/aarch64-apple-darwin/release/dora-coordinator archive_aarch64 cp target/aarch64-apple-darwin/release/dora-daemon archive_aarch64 - cp target/aarch64-apple-darwin/release/dora-cli archive_aarch64/dora + cp target/aarch64-apple-darwin/release/dora archive_aarch64/dora cd archive_aarch64 zip -r ../archive_aarch64.zip . cd .. From c6029d4f69e9c173e0c8d70426888db513502124 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 14:50:04 +0100 Subject: [PATCH 006/104] Fix python version --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 505a41d8..f2d4493b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,8 @@ jobs: # python examples - uses: actions/setup-python@v2 + with: + python-version: "3.8" - name: "Python Dataflow example" run: cargo run --example python-dataflow - name: "Python Operator Dataflow example" From 58a0853380583839c0d7b8733f356648a3d10d0f Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 15:08:23 +0100 Subject: [PATCH 007/104] Change cargo version --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- apis/python/node/dora/__init__.py | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8406497b..886678ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,7 +654,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "flume", "zenoh", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.0" +version = "0.3.1-rc2" [[package]] name = "concurrent-queue" @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "dora" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "bat", "clap 4.4.6", @@ -1415,7 +1415,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "arrow", "eyre", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "aligned-vec", "dora-message", @@ -1466,7 +1466,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "aligned-vec", "arrow-schema", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "eyre", "reqwest", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "arrow-data", "arrow-schema", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "arrow-array", "dora-node-api", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "cxx", "cxx-build", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "arrow", "dora-node-api", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1622,14 +1622,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "cxx", "cxx-build", @@ -1638,7 +1638,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "proc-macro2", "quote", @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "arrow", "dora-arrow-convert", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "chrono", "dora-node-api", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -3232,14 +3232,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4605,7 +4605,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4616,14 +4616,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4631,7 +4631,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.0" +version = "0.3.1-rc2" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index b2d0ff30..e939851e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.0" +version = "0.3.1-rc2" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.0", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.0", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.0", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.0", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.0", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.0", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.0", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.0", path = "apis/c/node" } -dora-core = { version = "0.3.0", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.0", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.0", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.0", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.0", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.0", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.0", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.0", path = "libraries/message" } -dora-runtime = { version = "0.3.0", path = "binaries/runtime" } -dora-daemon = { version = "0.3.0", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.0", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1-rc2", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1-rc2", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1-rc2", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1-rc2", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1-rc2", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1-rc2", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1-rc2", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1-rc2", path = "apis/c/node" } +dora-core = { version = "0.3.1-rc2", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1-rc2", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1-rc2", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1-rc2", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1-rc2", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1-rc2", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1-rc2", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1-rc2", path = "libraries/message" } +dora-runtime = { version = "0.3.1-rc2", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1-rc2", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1-rc2", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index aad31782..0bfce88c 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.0" +__version__ = "0.3.1-rc2" class DoraStatus(Enum): From 68ecf89b617795ec0f9e1bd3b8f2ce3d312fdd83 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 20:11:54 +0100 Subject: [PATCH 008/104] Remove explicit statement to `python3`and resolve it instead --- apis/python/node/README.md | 6 +++--- .../src/template/python/dataflow-template.yml | 2 +- examples/python-dataflow/run.rs | 18 +++++++++++++++--- examples/python-operator-dataflow/run.rs | 18 +++++++++++++++--- examples/python-ros2-dataflow/run.rs | 17 ++++++++++++++--- libraries/core/src/descriptor/validate.rs | 3 ++- libraries/core/src/lib.rs | 14 +++++++++++++- 7 files changed, 63 insertions(+), 15 deletions(-) diff --git a/apis/python/node/README.md b/apis/python/node/README.md index 408cd83c..5909eebe 100644 --- a/apis/python/node/README.md +++ b/apis/python/node/README.md @@ -4,9 +4,9 @@ This crate corresponds to the Node API for Dora. To build the Python module for development: -````bash -python3 -m venv .env +```bash +python -m venv .env source .env/bin/activate pip install maturin maturin develop -```` +``` diff --git a/binaries/cli/src/template/python/dataflow-template.yml b/binaries/cli/src/template/python/dataflow-template.yml index 3e7ed02c..2bebbc6f 100644 --- a/binaries/cli/src/template/python/dataflow-template.yml +++ b/binaries/cli/src/template/python/dataflow-template.yml @@ -16,7 +16,7 @@ nodes: - id: custom-node_1 custom: - source: python3 + source: python args: ./node_1/node_1.py inputs: tick: dora/timer/secs/1 diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index e33100b8..49721eb4 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::get_python_path; use eyre::{ContextCompat, WrapErr}; use std::path::Path; use tracing_subscriber::{ @@ -14,9 +15,20 @@ async fn main() -> eyre::Result<()> { std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - run(&["python3", "-m", "venv", "../.env"], None) - .await - .context("failed to create venv")?; + run( + &[ + get_python_path() + .context("Could not get python binary")? + .to_str() + .context("Could not convert python path to string")?, + "-m", + "venv", + "../.env", + ], + None, + ) + .await + .context("failed to create venv")?; let venv = &root.join("examples").join(".env"); std::env::set_var( "VIRTUAL_ENV", diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index e33100b8..49721eb4 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::get_python_path; use eyre::{ContextCompat, WrapErr}; use std::path::Path; use tracing_subscriber::{ @@ -14,9 +15,20 @@ async fn main() -> eyre::Result<()> { std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - run(&["python3", "-m", "venv", "../.env"], None) - .await - .context("failed to create venv")?; + run( + &[ + get_python_path() + .context("Could not get python binary")? + .to_str() + .context("Could not convert python path to string")?, + "-m", + "venv", + "../.env", + ], + None, + ) + .await + .context("failed to create venv")?; let venv = &root.join("examples").join(".env"); std::env::set_var( "VIRTUAL_ENV", diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index e531b8b2..a4ce5e91 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -14,9 +14,20 @@ async fn main() -> eyre::Result<()> { std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - run(&["python3", "-m", "venv", "../.env"], None) - .await - .context("failed to create venv")?; + run( + &[ + get_python_path() + .context("Could not get python binary")? + .to_str() + .context("Could not convert python path to string")?, + "-m", + "venv", + "../.env", + ], + None, + ) + .await + .context("failed to create venv")?; let venv = &root.join("examples").join(".env"); std::env::set_var( "VIRTUAL_ENV", diff --git a/libraries/core/src/descriptor/validate.rs b/libraries/core/src/descriptor/validate.rs index 6b0c599b..93063de9 100644 --- a/libraries/core/src/descriptor/validate.rs +++ b/libraries/core/src/descriptor/validate.rs @@ -2,6 +2,7 @@ use crate::{ adjust_shared_library_path, config::{DataId, Input, InputMapping, OperatorId, UserInputMapping}, descriptor::{self, source_is_url, CoreNodeKind, OperatorSource}, + get_python_path, }; use eyre::{bail, eyre, Context}; @@ -152,7 +153,7 @@ fn check_python_runtime() -> eyre::Result<()> { // Check if python dora-rs is installed and match cli version let reinstall_command = format!("Please reinstall it with: `pip install dora-rs=={VERSION} --force`"); - let mut command = Command::new("python3"); + let mut command = Command::new(get_python_path().context("Could not get python binary")?); command.args([ "-c", &format!( diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 786f5d80..9b6d81b8 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,4 +1,4 @@ -use eyre::{bail, eyre}; +use eyre::{bail, eyre, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, path::Path, @@ -30,3 +30,15 @@ pub fn adjust_shared_library_path(path: &Path) -> Result Result { + let python = match which::which("python3") { + Ok(python) => python, + Err(_) => which::which("python") + .context("failed to find `python` or `python3` in dora-daemon path. Make sure that python is available for the daemon.")?, + }; + Ok(python) +} From 0718c35265ddf927095405368d1cb5a2cd9d8fa5 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 20:17:20 +0100 Subject: [PATCH 009/104] Fix CD errors and try to release new version --- .github/workflows/release.yml | 10 ++--- Cargo.lock | 82 +++++++++++++++++------------------ Cargo.toml | 40 ++++++++--------- apis/python/node/Cargo.toml | 2 +- binaries/cli/Cargo.toml | 2 +- 5 files changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 148ef272..fc1fcd1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: cargo publish -p dora-node-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Publish binaries crates - cargo publish -p dora --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish -p dora-cli--token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-coordinator --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-runtime --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-daemon --token ${{ secrets.CARGO_REGISTRY_TOKEN }} @@ -83,7 +83,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora -p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora-cli-p dora-daemon" - name: Create Archive (Windows) if: runner.os == 'Windows' @@ -122,7 +122,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora -p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora-cli-p dora-daemon" - name: "Create Archive (Unix)" if: runner.os == 'Linux' || runner.os == 'macOS' @@ -166,7 +166,7 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora -p dora-daemon + args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli-p dora-daemon - name: "Archive Linux ARM64" if: runner.os == 'Linux' @@ -213,7 +213,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora -p dora-daemon + args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli-p dora-daemon - name: "Archive macOS ARM64" if: runner.os == 'macOS' diff --git a/Cargo.lock b/Cargo.lock index 886678ff..4d33ba52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,7 +654,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "flume", "zenoh", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1-rc2" +version = "0.3.1-rc3" [[package]] name = "concurrent-queue" @@ -1390,8 +1390,16 @@ dependencies = [ ] [[package]] -name = "dora" -version = "0.3.1-rc2" +name = "dora-arrow-convert" +version = "0.3.1-rc3" +dependencies = [ + "arrow", + "eyre", +] + +[[package]] +name = "dora-cli" +version = "0.3.1-rc3" dependencies = [ "bat", "clap 4.4.6", @@ -1413,17 +1421,9 @@ dependencies = [ "webbrowser", ] -[[package]] -name = "dora-arrow-convert" -version = "0.3.1-rc2" -dependencies = [ - "arrow", - "eyre", -] - [[package]] name = "dora-coordinator" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "aligned-vec", "dora-message", @@ -1466,7 +1466,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "aligned-vec", "arrow-schema", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "eyre", "reqwest", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "arrow-data", "arrow-schema", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "aligned-vec", "arrow", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "arrow-array", "dora-node-api", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "cxx", "cxx-build", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "arrow", "dora-node-api", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1622,14 +1622,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "cxx", "cxx-build", @@ -1638,7 +1638,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "proc-macro2", "quote", @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "aligned-vec", "arrow", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "arrow", "dora-arrow-convert", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "chrono", "dora-node-api", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "aligned-vec", "arrow", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -3232,14 +3232,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -4605,7 +4605,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -4616,14 +4616,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "eyre", @@ -4631,7 +4631,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1-rc2" +version = "0.3.1-rc3" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index e939851e..a5c049c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1-rc2" +version = "0.3.1-rc3" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1-rc2", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1-rc2", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1-rc2", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1-rc2", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1-rc2", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1-rc2", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1-rc2", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1-rc2", path = "apis/c/node" } -dora-core = { version = "0.3.1-rc2", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1-rc2", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1-rc2", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1-rc2", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1-rc2", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1-rc2", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1-rc2", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1-rc2", path = "libraries/message" } -dora-runtime = { version = "0.3.1-rc2", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1-rc2", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1-rc2", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1-rc3", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1-rc3", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1-rc3", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1-rc3", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1-rc3", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1-rc3", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1-rc3", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1-rc3", path = "apis/c/node" } +dora-core = { version = "0.3.1-rc3", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1-rc3", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1-rc3", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1-rc3", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1-rc3", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1-rc3", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1-rc3", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1-rc3", path = "libraries/message" } +dora-runtime = { version = "0.3.1-rc3", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1-rc3", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1-rc3", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/apis/python/node/Cargo.toml b/apis/python/node/Cargo.toml index 71bc9a10..3fbcccb8 100644 --- a/apis/python/node/Cargo.toml +++ b/apis/python/node/Cargo.toml @@ -28,4 +28,4 @@ dora-ros2-bridge-python = { workspace = true } [lib] name = "dora" -crate-type = ["lib", "cdylib"] +crate-type = ["cdylib"] diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index 004f296c..aa3b23f2 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "dora" +name = "dora-cli" version.workspace = true edition = "2021" documentation.workspace = true From 8bbdb25b90bcc5914009613bcbe22ba900e181cd Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 20:34:29 +0100 Subject: [PATCH 010/104] Fix CI compilation issue --- .github/workflows/ci.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0df63bc..1499edff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,41 @@ jobs: - name: "Check" run: cargo check --all - - name: "Build" + - name: "Build (UNIX)" + if: runner.os == 'Linux' || runner.os == 'macOS' run: cargo build --all + - name: "Build (Windows). Remove node python api as it create name collision." + if: runner.os == 'Windows' + run: | + + # build libraries crates + cargo build -p dora-message + cargo build -p dora-tracing + cargo build -p dora-metrics + cargo build -p dora-download + cargo build -p dora-core + cargo build -p communication-layer-pub-sub + cargo build -p communication-layer-request-reply + cargo build -p shared-memory-server + cargo build -p dora-arrow-convert + + # build rust API + cargo build -p dora-operator-api-macros + cargo build -p dora-operator-api-types + cargo build -p dora-operator-api + cargo build -p dora-node-api + # cargo build -p dora-operator-api-python + cargo build -p dora-operator-api-c + cargo build -p dora-node-api-c + + # build binaries crates + cargo build -p dora-cli + cargo build -p dora-coordinator + cargo build -p dora-runtime + cargo build -p dora-daemon + + # build extension crates + cargo build -p dora-record - name: "Test" # Remove Windows as there is `pdb` linker issue. # See: https://github.com/dora-rs/dora/pull/359#discussion_r1360268497 From c2a887edf3c5441bb0b362ae413257f60361179e Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 20:36:28 +0100 Subject: [PATCH 011/104] Nump to version 0.3.1-rc4 --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- apis/python/node/dora/__init__.py | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d33ba52..c57b02c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,7 +654,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "flume", "zenoh", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1-rc3" +version = "0.3.0-rc4" [[package]] name = "concurrent-queue" @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "arrow", "eyre", @@ -1399,7 +1399,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "bat", "clap 4.4.6", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "aligned-vec", "dora-message", @@ -1466,7 +1466,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "aligned-vec", "arrow-schema", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "eyre", "reqwest", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "arrow-data", "arrow-schema", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "arrow-array", "dora-node-api", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "cxx", "cxx-build", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "arrow", "dora-node-api", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1622,14 +1622,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "cxx", "cxx-build", @@ -1638,7 +1638,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "proc-macro2", "quote", @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "arrow", "dora-arrow-convert", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "chrono", "dora-node-api", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -3232,14 +3232,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4605,7 +4605,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4616,14 +4616,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4631,7 +4631,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1-rc3" +version = "0.3.0-rc4" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index a5c049c4..f140151b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1-rc3" +version = "0.3.0-rc4" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1-rc3", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1-rc3", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1-rc3", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1-rc3", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1-rc3", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1-rc3", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1-rc3", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1-rc3", path = "apis/c/node" } -dora-core = { version = "0.3.1-rc3", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1-rc3", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1-rc3", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1-rc3", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1-rc3", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1-rc3", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1-rc3", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1-rc3", path = "libraries/message" } -dora-runtime = { version = "0.3.1-rc3", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1-rc3", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1-rc3", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.0-rc4", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.0-rc4", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.0-rc4", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.0-rc4", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.0-rc4", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.0-rc4", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.0-rc4", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.0-rc4", path = "apis/c/node" } +dora-core = { version = "0.3.0-rc4", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.0-rc4", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.0-rc4", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.0-rc4", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.0-rc4", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.0-rc4", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.0-rc4", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.0-rc4", path = "libraries/message" } +dora-runtime = { version = "0.3.0-rc4", path = "binaries/runtime" } +dora-daemon = { version = "0.3.0-rc4", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.0-rc4", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 0bfce88c..1d15237f 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.1-rc2" +__version__ = "0.3.1-rc4" class DoraStatus(Enum): From 9413e4f19898d7622a62e2ba583a60671db6b95c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 20 Dec 2023 20:42:56 +0100 Subject: [PATCH 012/104] Rename `../.env` to `.env` as it seems to create path issues --- examples/python-dataflow/run.rs | 2 +- examples/python-operator-dataflow/run.rs | 2 +- examples/python-ros2-dataflow/run.rs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 49721eb4..da87b65f 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -23,7 +23,7 @@ async fn main() -> eyre::Result<()> { .context("Could not convert python path to string")?, "-m", "venv", - "../.env", + ".env", ], None, ) diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index 49721eb4..da87b65f 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -23,7 +23,7 @@ async fn main() -> eyre::Result<()> { .context("Could not convert python path to string")?, "-m", "venv", - "../.env", + ".env", ], None, ) diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index a4ce5e91..e01dcdd6 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::get_python_path; use eyre::{ContextCompat, WrapErr}; use std::path::Path; use tracing_subscriber::{ @@ -22,7 +23,7 @@ async fn main() -> eyre::Result<()> { .context("Could not convert python path to string")?, "-m", "venv", - "../.env", + ".env", ], None, ) From d97a2e16e91539ff91e278be4309ec9944b60157 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 21 Dec 2023 11:50:57 +0100 Subject: [PATCH 013/104] Fix typo --- .github/workflows/release.yml | 10 ++--- Cargo.lock | 66 +++++++++++++++--------------- Cargo.toml | 40 +++++++++--------- examples/benchmark/run.rs | 2 +- examples/multiple-daemons/run.rs | 2 +- examples/rust-dataflow-url/run.rs | 2 +- examples/rust-dataflow/run.rs | 2 +- examples/rust-ros2-dataflow/run.rs | 2 +- 8 files changed, 63 insertions(+), 63 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc1fcd1f..ce75ed88 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: cargo publish -p dora-node-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Publish binaries crates - cargo publish -p dora-cli--token ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish -p dora-cli --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-coordinator --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-runtime --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-daemon --token ${{ secrets.CARGO_REGISTRY_TOKEN }} @@ -83,7 +83,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli-p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" - name: Create Archive (Windows) if: runner.os == 'Windows' @@ -122,7 +122,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli-p dora-daemon" + run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" - name: "Create Archive (Unix)" if: runner.os == 'Linux' || runner.os == 'macOS' @@ -166,7 +166,7 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli-p dora-daemon + args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli -p dora-daemon - name: "Archive Linux ARM64" if: runner.os == 'Linux' @@ -213,7 +213,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli-p dora-daemon + args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli -p dora-daemon - name: "Archive macOS ARM64" if: runner.os == 'macOS' diff --git a/Cargo.lock b/Cargo.lock index c57b02c0..3a2040a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,7 +654,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "flume", "zenoh", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.0-rc4" +version = "0.3.1-rc4" [[package]] name = "concurrent-queue" @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "arrow", "eyre", @@ -1399,7 +1399,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "bat", "clap 4.4.6", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "aligned-vec", "dora-message", @@ -1466,7 +1466,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "aligned-vec", "arrow-schema", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "eyre", "reqwest", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "arrow-data", "arrow-schema", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "arrow-array", "dora-node-api", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "cxx", "cxx-build", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "arrow", "dora-node-api", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1622,14 +1622,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "cxx", "cxx-build", @@ -1638,7 +1638,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "proc-macro2", "quote", @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "arrow", "dora-arrow-convert", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "chrono", "dora-node-api", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "aligned-vec", "arrow", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -3232,14 +3232,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4605,7 +4605,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4616,14 +4616,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "eyre", @@ -4631,7 +4631,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.0-rc4" +version = "0.3.1-rc4" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index f140151b..de1efcc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.0-rc4" +version = "0.3.1-rc4" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.0-rc4", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.0-rc4", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.0-rc4", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.0-rc4", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.0-rc4", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.0-rc4", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.0-rc4", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.0-rc4", path = "apis/c/node" } -dora-core = { version = "0.3.0-rc4", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.0-rc4", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.0-rc4", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.0-rc4", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.0-rc4", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.0-rc4", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.0-rc4", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.0-rc4", path = "libraries/message" } -dora-runtime = { version = "0.3.0-rc4", path = "binaries/runtime" } -dora-daemon = { version = "0.3.0-rc4", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.0-rc4", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1-rc4", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1-rc4", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1-rc4", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1-rc4", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1-rc4", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1-rc4", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1-rc4", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1-rc4", path = "apis/c/node" } +dora-core = { version = "0.3.1-rc4", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1-rc4", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1-rc4", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1-rc4", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1-rc4", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1-rc4", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1-rc4", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1-rc4", path = "libraries/message" } +dora-runtime = { version = "0.3.1-rc4", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1-rc4", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1-rc4", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/examples/benchmark/run.rs b/examples/benchmark/run.rs index 834f4ebb..a7d35429 100644 --- a/examples/benchmark/run.rs +++ b/examples/benchmark/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora"); + cmd.arg("--package").arg("dora-cli"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index f47416e3..f1ee34de 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -204,7 +204,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora"); + cmd.arg("--package").arg("dora-cli"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-dataflow-url/run.rs b/examples/rust-dataflow-url/run.rs index 0b25d79d..dc27e137 100644 --- a/examples/rust-dataflow-url/run.rs +++ b/examples/rust-dataflow-url/run.rs @@ -34,7 +34,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora"); + cmd.arg("--package").arg("dora-cli"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-dataflow/run.rs b/examples/rust-dataflow/run.rs index 834f4ebb..a7d35429 100644 --- a/examples/rust-dataflow/run.rs +++ b/examples/rust-dataflow/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora"); + cmd.arg("--package").arg("dora-cli"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); diff --git a/examples/rust-ros2-dataflow/run.rs b/examples/rust-ros2-dataflow/run.rs index 834f4ebb..a7d35429 100644 --- a/examples/rust-ros2-dataflow/run.rs +++ b/examples/rust-ros2-dataflow/run.rs @@ -35,7 +35,7 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); - cmd.arg("--package").arg("dora"); + cmd.arg("--package").arg("dora-cli"); cmd.arg("--").arg("build").arg(dataflow); if !cmd.status().await?.success() { bail!("failed to build dataflow"); From a2de951d98492ebafd007e182e9e6ad04fe460f7 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 21 Dec 2023 12:31:35 +0100 Subject: [PATCH 014/104] Remove setup tracing from examples to make it more simple --- Cargo.lock | 1 + Cargo.toml | 1 + examples/benchmark/run.rs | 16 ++--------- examples/c++-dataflow/run.rs | 16 ++--------- examples/c-dataflow/run.rs | 16 ++--------- examples/cmake-dataflow/run.rs | 16 ++--------- examples/multiple-daemons/run.rs | 16 ++--------- examples/python-dataflow/run.rs | 35 ++---------------------- examples/python-operator-dataflow/run.rs | 35 ++---------------------- examples/python-ros2-dataflow/run.rs | 35 ++---------------------- examples/rust-dataflow-url/run.rs | 16 ++--------- examples/rust-dataflow/run.rs | 16 ++--------- examples/rust-ros2-dataflow/run.rs | 16 ++--------- 13 files changed, 24 insertions(+), 211 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ad9cc27..4208d6e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1512,6 +1512,7 @@ dependencies = [ "dora-coordinator", "dora-core", "dora-daemon", + "dora-tracing", "dunce", "eyre", "futures", diff --git a/Cargo.toml b/Cargo.toml index b2d0ff30..15a5334b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ tokio = "1.24.2" dora-daemon = { workspace = true } dora-coordinator = { workspace = true } dora-core = { workspace = true } +dora-tracing = { workspace = true } dunce = "1.0.2" serde_yaml = "0.8.23" uuid = { version = "1.2.1", features = ["v4", "serde"] } diff --git a/examples/benchmark/run.rs b/examples/benchmark/run.rs index a7d35429..54c8ac76 100644 --- a/examples/benchmark/run.rs +++ b/examples/benchmark/run.rs @@ -1,7 +1,6 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -17,7 +16,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing("benchmark-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -42,14 +41,3 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 27914ac3..6915c73f 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -1,10 +1,9 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX}, path::Path, }; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -19,7 +18,7 @@ async fn main() -> eyre::Result<()> { if run_dora_runtime { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing")?; + set_up_tracing("c++-dataflow-runner").wrap_err("failed to set up tracing")?; if cfg!(windows) { tracing::error!( @@ -294,14 +293,3 @@ async fn build_cxx_operator( Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/c-dataflow/run.rs b/examples/c-dataflow/run.rs index 7db5a7f7..4268c25c 100644 --- a/examples/c-dataflow/run.rs +++ b/examples/c-dataflow/run.rs @@ -1,10 +1,9 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX}, path::Path, }; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -20,7 +19,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing")?; + set_up_tracing("c-dataflow-runner").wrap_err("failed to set up tracing")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -182,14 +181,3 @@ async fn build_c_operator(root: &Path) -> eyre::Result<()> { Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/cmake-dataflow/run.rs b/examples/cmake-dataflow/run.rs index a3c0cdfb..6ede3e72 100644 --- a/examples/cmake-dataflow/run.rs +++ b/examples/cmake-dataflow/run.rs @@ -1,7 +1,6 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -16,7 +15,7 @@ async fn main() -> eyre::Result<()> { if run_dora_runtime { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing")?; + set_up_tracing("cmake-dataflow-runner").wrap_err("failed to set up tracing")?; if cfg!(windows) { tracing::error!( @@ -67,14 +66,3 @@ async fn build_package(package: &str) -> eyre::Result<()> { } Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index f1ee34de..d3499382 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -3,6 +3,7 @@ use dora_core::{ descriptor::Descriptor, topics::{ControlRequest, ControlRequestReply, DataflowId}, }; +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use futures::stream; use std::{ @@ -19,8 +20,6 @@ use tokio::{ task::JoinSet, }; use tokio_stream::wrappers::ReceiverStream; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; use uuid::Uuid; #[derive(Debug, Clone, clap::Parser)] @@ -36,7 +35,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing("multiple-daemon-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -211,14 +210,3 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::TRACE); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index da87b65f..e95aee8a 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,15 +1,10 @@ -use dora_core::get_python_path; +use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; -use tracing_subscriber::{ - filter::{FilterExt, LevelFilter}, - prelude::*, - EnvFilter, Registry, -}; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing()?; + set_up_tracing("python-dataflow-runner")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -63,29 +58,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> { - let mut run = tokio::process::Command::new(cmd[0]); - run.args(&cmd[1..]); - - if let Some(pwd) = pwd { - run.current_dir(pwd); - } - if !run.status().await?.success() { - eyre::bail!("failed to run {cmd:?}"); - }; - Ok(()) -} - -pub fn set_up_tracing() -> eyre::Result<()> { - // Filter log using `RUST_LOG`. More useful for CLI. - let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG); - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(filter); - - let registry = Registry::default().with(stdout_log); - - tracing::subscriber::set_global_default(registry) - .context("failed to set tracing global subscriber") -} diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index da87b65f..efe6ff09 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,15 +1,10 @@ -use dora_core::get_python_path; +use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; -use tracing_subscriber::{ - filter::{FilterExt, LevelFilter}, - prelude::*, - EnvFilter, Registry, -}; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing()?; + set_up_tracing("python-operator-dataflow-runner")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -63,29 +58,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> { - let mut run = tokio::process::Command::new(cmd[0]); - run.args(&cmd[1..]); - - if let Some(pwd) = pwd { - run.current_dir(pwd); - } - if !run.status().await?.success() { - eyre::bail!("failed to run {cmd:?}"); - }; - Ok(()) -} - -pub fn set_up_tracing() -> eyre::Result<()> { - // Filter log using `RUST_LOG`. More useful for CLI. - let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG); - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(filter); - - let registry = Registry::default().with(stdout_log); - - tracing::subscriber::set_global_default(registry) - .context("failed to set tracing global subscriber") -} diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index e01dcdd6..5d873f02 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,15 +1,10 @@ -use dora_core::get_python_path; +use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; -use tracing_subscriber::{ - filter::{FilterExt, LevelFilter}, - prelude::*, - EnvFilter, Registry, -}; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing()?; + set_up_tracing("python-ros2-dataflow-runner")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -65,29 +60,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> { - let mut run = tokio::process::Command::new(cmd[0]); - run.args(&cmd[1..]); - - if let Some(pwd) = pwd { - run.current_dir(pwd); - } - if !run.status().await?.success() { - eyre::bail!("failed to run {cmd:?}"); - }; - Ok(()) -} - -pub fn set_up_tracing() -> eyre::Result<()> { - // Filter log using `RUST_LOG`. More useful for CLI. - let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG); - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(filter); - - let registry = Registry::default().with(stdout_log); - - tracing::subscriber::set_global_default(registry) - .context("failed to set tracing global subscriber") -} diff --git a/examples/rust-dataflow-url/run.rs b/examples/rust-dataflow-url/run.rs index dc27e137..7e16d3b9 100644 --- a/examples/rust-dataflow-url/run.rs +++ b/examples/rust-dataflow-url/run.rs @@ -1,7 +1,6 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -16,7 +15,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing")?; + set_up_tracing("rust-dataflow-url-runner").wrap_err("failed to set up tracing")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -41,14 +40,3 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/rust-dataflow/run.rs b/examples/rust-dataflow/run.rs index a7d35429..2fbc50d9 100644 --- a/examples/rust-dataflow/run.rs +++ b/examples/rust-dataflow/run.rs @@ -1,7 +1,6 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -17,7 +16,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing("rust-dataflow-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -42,14 +41,3 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} diff --git a/examples/rust-ros2-dataflow/run.rs b/examples/rust-ros2-dataflow/run.rs index a7d35429..cb999d8c 100644 --- a/examples/rust-ros2-dataflow/run.rs +++ b/examples/rust-ros2-dataflow/run.rs @@ -1,7 +1,6 @@ +use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -use tracing::metadata::LevelFilter; -use tracing_subscriber::Layer; #[derive(Debug, Clone, clap::Parser)] pub struct Args { @@ -17,7 +16,7 @@ async fn main() -> eyre::Result<()> { return tokio::task::block_in_place(dora_daemon::run_dora_runtime); } - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing("rust-ros2-dataflow-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) @@ -42,14 +41,3 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } - -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; - - let stdout_log = tracing_subscriber::fmt::layer() - .pretty() - .with_filter(LevelFilter::DEBUG); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) - .context("failed to set tracing global subscriber") -} From 0a60246038412577cac646d8ab4ee5dbada4ab31 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 21 Dec 2023 12:32:06 +0100 Subject: [PATCH 015/104] Make `run` a helper function and use OsStr to search for path --- examples/python-dataflow/run.rs | 35 +++++++++++---------- examples/python-operator-dataflow/run.rs | 35 +++++++++++---------- examples/python-ros2-dataflow/run.rs | 39 ++++++++++++----------- libraries/core/Cargo.toml | 2 +- libraries/core/src/lib.rs | 40 +++++++++++++++++++++--- 5 files changed, 94 insertions(+), 57 deletions(-) diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index e95aee8a..74fcd1fb 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -39,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index efe6ff09..7554a5e2 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -39,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 5d873f02..60ecc106 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run, set_up_tracing}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -27,9 +21,7 @@ async fn main() -> eyre::Result<()> { let venv = &root.join("examples").join(".env"); std::env::set_var( "VIRTUAL_ENV", - venv.to_str() - .context("venv path not valid unicode")? - .to_owned(), + venv.to_str().context("venv path not valid unicode")?, ); let orig_path = std::env::var("PATH")?; let venv_bin = venv.join("bin"); @@ -41,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index 3be48e23..c5961e1d 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -19,5 +19,5 @@ uuid = { version = "1.2.1", features = ["serde"] } dora-message = { workspace = true } tracing = "0.1" serde-with-expand-env = "1.1.0" -tokio = { version = "1.24.1", features = ["fs"] } +tokio = { version = "1.24.1", features = ["fs", "process"] } aligned-vec = { version = "0.5.0", features = ["serde"] } diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 9b6d81b8..e0145d2b 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,6 +1,7 @@ use eyre::{bail, eyre, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, + ffi::OsStr, path::Path, }; @@ -33,12 +34,41 @@ pub fn adjust_shared_library_path(path: &Path) -> Result Result { let python = match which::which("python3") { - Ok(python) => python, - Err(_) => which::which("python") - .context("failed to find `python` or `python3` in dora-daemon path. Make sure that python is available for the daemon.")?, - }; + Ok(python) => python, + Err(_) => which::which("python") + .context("failed to find `python` or `python3`. Make sure that python is available.")?, + }; Ok(python) } + +// Search for pip binary. +// First search for `pip3` as for ubuntu <20, `pip` can resolves to `python2,7 -m pip` +// Then search for `pip`, this will resolve for windows to python3 -m pip. +pub fn get_pip_path() -> Result { + let python = match which::which("pip3") { + Ok(python) => python, + Err(_) => which::which("pip") + .context("failed to find `pip3` or `pip`. Make sure that python is available.")?, + }; + Ok(python) +} + +// Helper function to run a program +pub async fn run(program: S, args: &[&str], pwd: Option<&Path>) -> eyre::Result<()> +where + S: AsRef, +{ + let mut run = tokio::process::Command::new(program); + run.args(args); + + if let Some(pwd) = pwd { + run.current_dir(pwd); + } + if !run.status().await?.success() { + eyre::bail!("failed to run {args:?}"); + }; + Ok(()) +} From 07b3c99d1d110fdff53696abd3fcef3b4f37c095 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:31:55 +0000 Subject: [PATCH 016/104] Bump unsafe-libyaml from 0.2.9 to 0.2.10 Bumps [unsafe-libyaml](https://github.com/dtolnay/unsafe-libyaml) from 0.2.9 to 0.2.10. - [Release notes](https://github.com/dtolnay/unsafe-libyaml/releases) - [Commits](https://github.com/dtolnay/unsafe-libyaml/compare/0.2.9...0.2.10) --- updated-dependencies: - dependency-name: unsafe-libyaml dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f074414b..b7728668 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5783,9 +5783,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" [[package]] name = "untrusted" From 2224971a34626a24c6c910f1e7c847020962cf29 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 22 Dec 2023 10:18:24 +0100 Subject: [PATCH 017/104] Fix ROS2 error --- Cargo.lock | 4 ++-- examples/python-ros2-dataflow/run.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4208d6e6..03812f2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,9 +1241,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.107" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ce20f6b8433da4841b1dadfb9468709868022d829d5ca1f2ffbda928455ea3" +checksum = "51bc81d2664db24cf1d35405f66e18a85cffd4d49ab930c71a5c6342a410f38c" dependencies = [ "cc", "codespan-reporting", diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 60ecc106..26f61224 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,4 +1,4 @@ -use dora_core::{get_pip_path, get_python_path, run, set_up_tracing}; +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; From 808cd86f4b772d83d62105d284f55e0815d7506a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 22 Dec 2023 10:20:44 +0100 Subject: [PATCH 018/104] Use `--exclude` flag to remove `dora-node-api-python` --- .github/workflows/ci.yml | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1499edff..69e3a217 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,41 +62,8 @@ jobs: - name: "Check" run: cargo check --all - - name: "Build (UNIX)" - if: runner.os == 'Linux' || runner.os == 'macOS' - run: cargo build --all - - name: "Build (Windows). Remove node python api as it create name collision." - if: runner.os == 'Windows' - run: | - - # build libraries crates - cargo build -p dora-message - cargo build -p dora-tracing - cargo build -p dora-metrics - cargo build -p dora-download - cargo build -p dora-core - cargo build -p communication-layer-pub-sub - cargo build -p communication-layer-request-reply - cargo build -p shared-memory-server - cargo build -p dora-arrow-convert - - # build rust API - cargo build -p dora-operator-api-macros - cargo build -p dora-operator-api-types - cargo build -p dora-operator-api - cargo build -p dora-node-api - # cargo build -p dora-operator-api-python - cargo build -p dora-operator-api-c - cargo build -p dora-node-api-c - - # build binaries crates - cargo build -p dora-cli - cargo build -p dora-coordinator - cargo build -p dora-runtime - cargo build -p dora-daemon - - # build extension crates - cargo build -p dora-record + - name: "Build (Without Python node as it is build with maturin)" + run: cargo build --all --exclude dora-node-api-python - name: "Test" # Remove Windows as there is `pdb` linker issue. # See: https://github.com/dora-rs/dora/pull/359#discussion_r1360268497 From 3a1c8fca2e3e291291b18a74738144e4024fcc11 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 22 Dec 2023 10:33:22 +0100 Subject: [PATCH 019/104] Remove aliasing in documentation --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7be33f38..78f1f4ea 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,6 @@ Quickest way: ```bash cargo install dora-cli -alias dora='dora-cli' cargo install dora-coordinator cargo install dora-daemon pip install dora-rs ## For Python API From eab690d199c9f159ba77f4c0656f83d489ca7c53 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 2 Jan 2024 15:10:22 +0100 Subject: [PATCH 020/104] Revert to 0.3.0 --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- apis/python/node/dora/__init__.py | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a2040a5..f074414b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,7 +654,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "flume", "zenoh", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1-rc4" +version = "0.3.0" [[package]] name = "concurrent-queue" @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "arrow", "eyre", @@ -1399,7 +1399,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "bat", "clap 4.4.6", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "aligned-vec", "dora-message", @@ -1466,7 +1466,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "aligned-vec", "arrow-schema", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "eyre", "reqwest", @@ -1524,7 +1524,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "arrow-data", "arrow-schema", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "aligned-vec", "arrow", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "arrow-array", "dora-node-api", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "cxx", "cxx-build", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "arrow", "dora-node-api", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1622,14 +1622,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "cxx", "cxx-build", @@ -1638,7 +1638,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "proc-macro2", "quote", @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "aligned-vec", "arrow", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "arrow", "dora-arrow-convert", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "chrono", "dora-node-api", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "aligned-vec", "arrow", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -3232,14 +3232,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -4605,7 +4605,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -4616,14 +4616,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "eyre", @@ -4631,7 +4631,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1-rc4" +version = "0.3.0" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index de1efcc2..b2d0ff30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1-rc4" +version = "0.3.0" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1-rc4", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1-rc4", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1-rc4", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1-rc4", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1-rc4", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1-rc4", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1-rc4", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1-rc4", path = "apis/c/node" } -dora-core = { version = "0.3.1-rc4", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1-rc4", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1-rc4", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1-rc4", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1-rc4", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1-rc4", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1-rc4", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1-rc4", path = "libraries/message" } -dora-runtime = { version = "0.3.1-rc4", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1-rc4", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1-rc4", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.0", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.0", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.0", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.0", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.0", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.0", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.0", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.0", path = "apis/c/node" } +dora-core = { version = "0.3.0", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.0", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.0", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.0", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.0", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.0", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.0", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.0", path = "libraries/message" } +dora-runtime = { version = "0.3.0", path = "binaries/runtime" } +dora-daemon = { version = "0.3.0", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.0", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 1d15237f..aad31782 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.1-rc4" +__version__ = "0.3.0" class DoraStatus(Enum): From 56169862727222ec28c5e992d2fab1b728a94b28 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 2 Jan 2024 15:59:13 +0100 Subject: [PATCH 021/104] Run windows program using cmd.exe to wrapped_args) to instantiate powershell. This is required for python environment --- libraries/core/src/lib.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index e0145d2b..cd4d31c3 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,4 +1,4 @@ -use eyre::{bail, eyre, Context}; +use eyre::{bail, eyre, Context, ContextCompat}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, ffi::OsStr, @@ -61,8 +61,24 @@ pub async fn run(program: S, args: &[&str], pwd: Option<&Path>) -> eyre::Resu where S: AsRef, { - let mut run = tokio::process::Command::new(program); - run.args(args); + // if platform is windows, use a shell + let mut run = if cfg!(windows) { + let mut run = tokio::process::Command::new("cmd.exe"); + let mut wrapped_args = vec![ + "/c", + program + .as_ref() + .to_str() + .context("Could not get path string")?, + ]; + wrapped_args.extend(args); + run.args(wrapped_args); + run + } else { + let mut run = tokio::process::Command::new(program); + run.args(args); + run + }; if let Some(pwd) = pwd { run.current_dir(pwd); From 46e56da46962f2cc87b4faab82083391a0a28fff Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 2 Jan 2024 16:56:03 +0100 Subject: [PATCH 022/104] Uses python 3.10 that fixes an error on windows: https://github.com/actions/runner-images/issues/2690 --- .github/workflows/ci.yml | 10 ++++++++++ libraries/core/src/lib.rs | 20 ++------------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2d4493b..1af7e115 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,8 +132,13 @@ jobs: # python examples - uses: actions/setup-python@v2 + if: runner.os != 'Windows' with: python-version: "3.8" + - uses: actions/setup-python@v2 + if: runner.os == 'Windows' + with: + python-version: "3.10" - name: "Python Dataflow example" run: cargo run --example python-dataflow - name: "Python Operator Dataflow example" @@ -167,8 +172,13 @@ jobs: source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node & cargo run --example rust-ros2-dataflow --features="ros2-examples" - uses: actions/setup-python@v2 + if: runner.os != 'Windows' with: python-version: "3.8" + - uses: actions/setup-python@v2 + if: runner.os == 'Windows' + with: + python-version: "3.10" - name: "python-ros2-dataflow" timeout-minutes: 30 env: diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index cd4d31c3..4d1420ec 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -61,24 +61,8 @@ pub async fn run(program: S, args: &[&str], pwd: Option<&Path>) -> eyre::Resu where S: AsRef, { - // if platform is windows, use a shell - let mut run = if cfg!(windows) { - let mut run = tokio::process::Command::new("cmd.exe"); - let mut wrapped_args = vec![ - "/c", - program - .as_ref() - .to_str() - .context("Could not get path string")?, - ]; - wrapped_args.extend(args); - run.args(wrapped_args); - run - } else { - let mut run = tokio::process::Command::new(program); - run.args(args); - run - }; + let mut run = tokio::process::Command::new(program); + run.args(args); if let Some(pwd) = pwd { run.current_dir(pwd); From c87bb1b2838531a3ef54f70ddad98ee0362811ad Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 3 Jan 2024 11:34:54 +0100 Subject: [PATCH 023/104] Use python to execute python script --- binaries/daemon/src/spawn.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index e1e25a77..662c6bc5 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -8,6 +8,7 @@ use dora_core::{ descriptor::{ resolve_path, source_is_url, Descriptor, OperatorSource, ResolvedNode, SHELL_SOURCE, }, + get_python_path, message::uhlc::HLC, }; use dora_download::download_file; @@ -82,7 +83,21 @@ pub async fn spawn_node( }; tracing::info!("spawning {}", resolved_path.display()); - let mut cmd = tokio::process::Command::new(&resolved_path); + // If extension is .py, use python to run the script + let mut cmd = match resolved_path.extension().map(|ext| ext.to_str()) { + Some(Some("py")) => { + let mut cmd = tokio::process::Command::new( + &get_python_path().context("Could not get python path")?, + ); + cmd.arg(&resolved_path); + cmd + } + _ => { + let cmd = tokio::process::Command::new(&resolved_path); + cmd + } + }; + if let Some(args) = &n.args { cmd.args(args.split_ascii_whitespace()); } From 06ea8ce0b267aa9d69589c3bcd3394ee11d22d72 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 3 Jan 2024 17:01:46 +0100 Subject: [PATCH 024/104] Make turtle movement detection more robust Compare min and max positions instead of using assert with initial position (which can fail spuriously if the first pose arrives before the first movement; and also when the turtle reaches the exact same position again). --- examples/python-ros2-dataflow/random_turtle.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/python-ros2-dataflow/random_turtle.py b/examples/python-ros2-dataflow/random_turtle.py index d466ab1e..3e97228b 100755 --- a/examples/python-ros2-dataflow/random_turtle.py +++ b/examples/python-ros2-dataflow/random_turtle.py @@ -39,6 +39,12 @@ dora_node.merge_external_events(pose_reader) print("looping", flush=True) +# take track of minimum and maximum coordinates of turtle +min_x = 1000 +max_x = 0 +min_y = 1000 +max_y = 0 + for i in range(500): event = dora_node.next() if event is None: @@ -55,8 +61,10 @@ for i in range(500): # ROS2 Event elif event_kind == "external": pose = event.inner()[0].as_py() - if i == CHECK_TICK: - assert ( - pose["x"] != 5.544444561004639 - ), "turtle should not be at initial x axis" + min_x = min([min_x, pose["x"]]) + max_x = max([max_x, pose["x"]]) + min_y = min([min_y, pose["y"]]) + max_y = max([max_y, pose["y"]]) dora_node.send_output("turtle_pose", event.inner()) + +assert max_x - min_x > 1 or max_y - min_y > 1, "no turtle movement" From b5ce87140e7211370c77f7403d0df8c71d5dbf70 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 3 Jan 2024 17:33:25 +0100 Subject: [PATCH 025/104] Increase timeout of ROS2 bridge examples CI job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99c2a214..46da5fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: ros2-bridge-examples: name: "ROS2 Bridge Examples" runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 From 06a7d0bf0c56bab105a38225c68a091e23531521 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 3 Jan 2024 19:48:47 +0100 Subject: [PATCH 026/104] Fix read error in C++ node input Thwy were an error of typecasting within the C++ node input. This fixes #403 I have not adressed all input type as I think a solution based on arrow FFI would be better. --- apis/c/node/src/lib.rs | 28 +++++++++++++++------------- examples/c++-dataflow/dataflow.yml | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/apis/c/node/src/lib.rs b/apis/c/node/src/lib.rs index 88a77675..d8885b85 100644 --- a/apis/c/node/src/lib.rs +++ b/apis/c/node/src/lib.rs @@ -1,6 +1,6 @@ #![deny(unsafe_op_in_unsafe_fn)] -use arrow_array::BinaryArray; +use arrow_array::UInt8Array; use dora_node_api::{arrow::array::AsArray, DoraNode, Event, EventStream}; use eyre::Context; use std::{ffi::c_void, ptr, slice}; @@ -170,22 +170,24 @@ pub unsafe extern "C" fn read_dora_input_data( ) { let event: &Event = unsafe { &*event.cast() }; match event { - Event::Input { data, .. } => { - let data: Option<&BinaryArray> = data.as_binary_opt(); - if let Some(data) = data { - let ptr = data.value(0).as_ptr(); - let len = data.value(0).len(); + Event::Input { data, metadata, .. } => match metadata.type_info.data_type { + dora_node_api::arrow::datatypes::DataType::UInt8 => { + let array: &UInt8Array = data.as_primitive(); + let ptr = array.values().as_ptr(); unsafe { *out_ptr = ptr; - *out_len = len; - } - } else { - unsafe { - *out_ptr = ptr::null(); - *out_len = 0; + *out_len = metadata.type_info.len; } } - } + dora_node_api::arrow::datatypes::DataType::Null => unsafe { + *out_ptr = ptr::null(); + *out_len = 0; + }, + _ => { + todo!("dora C++ Node does not yet support higher level type of arrow. Only UInt8. + The ultimate solution should be based on arrow FFI interface. Feel free to contribute :)") + } + }, _ => unsafe { *out_ptr = ptr::null(); *out_len = 0; diff --git a/examples/c++-dataflow/dataflow.yml b/examples/c++-dataflow/dataflow.yml index 84c743a8..d2186dd8 100644 --- a/examples/c++-dataflow/dataflow.yml +++ b/examples/c++-dataflow/dataflow.yml @@ -10,7 +10,7 @@ nodes: custom: source: build/node_c_api inputs: - tick: dora/timer/millis/300 + tick: cxx-node-rust-api/counter outputs: - counter From 4462d99eb4bd2a6d8b72bde8971475b804516cf1 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 5 Jan 2024 15:59:59 +0100 Subject: [PATCH 027/104] Add pythpn version to debug message --- binaries/daemon/src/spawn.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 662c6bc5..e17deff5 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -133,9 +133,10 @@ pub async fn spawn_node( .spawn() .wrap_err_with(move || { format!( - "failed to run `{}` with args `{}`", + "failed to run `{}` with args `{}`. If this was run with python, the python used was: {:?}", n.source, - n.args.as_deref().unwrap_or_default() + n.args.as_deref().unwrap_or_default(), + get_python_path().unwrap_or_default() ) })? } From c311bc12304f2ab1d6d66651824b21d8a52ed499 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 5 Jan 2024 16:27:13 +0100 Subject: [PATCH 028/104] Log which python is used --- binaries/daemon/src/spawn.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index e17deff5..2e731c01 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -82,17 +82,17 @@ pub async fn spawn_node( })? }; - tracing::info!("spawning {}", resolved_path.display()); // If extension is .py, use python to run the script let mut cmd = match resolved_path.extension().map(|ext| ext.to_str()) { Some(Some("py")) => { - let mut cmd = tokio::process::Command::new( - &get_python_path().context("Could not get python path")?, - ); + let python = get_python_path().context("Could not get python path")?; + tracing::info!("spawning: {:?} {}", &python, resolved_path.display()); + let mut cmd = tokio::process::Command::new(&python); cmd.arg(&resolved_path); cmd } _ => { + tracing::info!("spawning: {}", resolved_path.display()); let cmd = tokio::process::Command::new(&resolved_path); cmd } @@ -133,10 +133,9 @@ pub async fn spawn_node( .spawn() .wrap_err_with(move || { format!( - "failed to run `{}` with args `{}`. If this was run with python, the python used was: {:?}", + "failed to run `{}` with args `{}.", n.source, n.args.as_deref().unwrap_or_default(), - get_python_path().unwrap_or_default() ) })? } @@ -153,11 +152,7 @@ pub async fn spawn_node( let mut command = if has_python_operator && !has_other_operator { // Use python to spawn runtime if there is a python operator - let python = match which::which("python3") { - Ok(python) => python, - Err(_) => which::which("python") - .context("failed to find `python` or `python3` in dora-daemon path. Make sure that python is available for the daemon.")?, - }; + let python = get_python_path().context("Could not find python in daemon")?; let mut command = tokio::process::Command::new(python); command.args([ "-c", From 20c02089b6f279d1bb49c49be9abb94e71d7d6ef Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 14:33:03 +0100 Subject: [PATCH 029/104] Fix Python CI on windows --- .github/workflows/ci.yml | 2 +- examples/python-dataflow/run.rs | 8 +++++++- examples/python-operator-dataflow/run.rs | 8 +++++++- examples/python-ros2-dataflow/run.rs | 8 +++++++- libraries/core/src/lib.rs | 2 +- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1af7e115..f677d0e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: env: - RUST_LOG: trace + RUST_LOG: INFO jobs: test: diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 74fcd1fb..16e9d2d6 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -24,7 +24,13 @@ async fn main() -> eyre::Result<()> { venv.to_str().context("venv path not valid unicode")?, ); let orig_path = std::env::var("PATH")?; - let venv_bin = venv.join("bin"); + // bin folder is named Scripts on windows. + // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 + let venv_bin = if cfg!(windows) { + venv.join("Scripts") + } else { + venv.join("bin") + }; std::env::set_var( "PATH", format!( diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index 7554a5e2..4cb19c6d 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -24,7 +24,13 @@ async fn main() -> eyre::Result<()> { venv.to_str().context("venv path not valid unicode")?, ); let orig_path = std::env::var("PATH")?; - let venv_bin = venv.join("bin"); + // bin folder is named Scripts on windows. + // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 + let venv_bin = if cfg!(windows) { + venv.join("Scripts") + } else { + venv.join("bin") + }; std::env::set_var( "PATH", format!( diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 26f61224..f136ae46 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -24,7 +24,13 @@ async fn main() -> eyre::Result<()> { venv.to_str().context("venv path not valid unicode")?, ); let orig_path = std::env::var("PATH")?; - let venv_bin = venv.join("bin"); + // bin folder is named Scripts on windows. + // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 + let venv_bin = if cfg!(windows) { + venv.join("Scripts") + } else { + venv.join("bin") + }; std::env::set_var( "PATH", format!( diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 4d1420ec..e0145d2b 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,4 +1,4 @@ -use eyre::{bail, eyre, Context, ContextCompat}; +use eyre::{bail, eyre, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, ffi::OsStr, From 0870f775ee2a73dedd9d4b5f9c4a6b63d3f7c8ac Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 14:48:53 +0100 Subject: [PATCH 030/104] Remove introduced typo in PR --- binaries/cli/src/template/python/dataflow-template.yml | 3 +-- binaries/daemon/src/spawn.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/binaries/cli/src/template/python/dataflow-template.yml b/binaries/cli/src/template/python/dataflow-template.yml index 2bebbc6f..782d78a6 100644 --- a/binaries/cli/src/template/python/dataflow-template.yml +++ b/binaries/cli/src/template/python/dataflow-template.yml @@ -16,8 +16,7 @@ nodes: - id: custom-node_1 custom: - source: python - args: ./node_1/node_1.py + source: ./node_1/node_1.py inputs: tick: dora/timer/secs/1 input-1: op_1/some-output diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 2e731c01..1a7e54c3 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -133,7 +133,7 @@ pub async fn spawn_node( .spawn() .wrap_err_with(move || { format!( - "failed to run `{}` with args `{}.", + "failed to run `{}` with args `{}`", n.source, n.args.as_deref().unwrap_or_default(), ) From 82f27949160f0a36337007a0209c45b4118b84ef Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 15:46:11 +0100 Subject: [PATCH 031/104] Only use python for windows and python3 for linux --- libraries/core/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index e0145d2b..83ce0dc5 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -33,13 +33,14 @@ pub fn adjust_shared_library_path(path: &Path) -> Result Result { - let python = match which::which("python3") { - Ok(python) => python, - Err(_) => which::which("python") - .context("failed to find `python` or `python3`. Make sure that python is available.")?, + let python = if cfg!(windows) { + which::which("python") + .context("failed to find `python` or `python3`. Make sure that python is available.")? + } else { + which::which("python3") + .context("failed to find `python` or `python3`. Make sure that python is available.")? }; Ok(python) } From ea1259da2048a32544dc92f3bb93eb0f21c22e73 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 16:45:38 +0100 Subject: [PATCH 032/104] Bump which version --- Cargo.lock | 1272 ++++++++++++++++++++++++++++--- binaries/coordinator/Cargo.toml | 2 +- binaries/daemon/Cargo.toml | 2 +- libraries/core/Cargo.toml | 2 +- libraries/core/src/lib.rs | 3 +- 5 files changed, 1169 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03812f2b..3f7c0215 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,15 +30,16 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -59,6 +60,12 @@ dependencies = [ "serde", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -193,7 +200,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "num", ] @@ -317,7 +324,7 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.3", ] [[package]] @@ -511,7 +518,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -794,18 +801,168 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cab" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6b4de23c7d39c0631fd3cc952d87951c86c75a13812d7247cb7a896e7b3551" +dependencies = [ + "byteorder", + "flate2", + "lzxd", + "time", +] + [[package]] name = "cache-padded" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + [[package]] name = "capnp" version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dca085c2c7d9d65ad749d450b19b551efaa8e3476a439bdca07aca8533097f3" +[[package]] +name = "cargo-config2" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90811f7e462b86622e20549c1c43bb3861d1bf011a08c2ceceaeb51fdfc27402" +dependencies = [ + "home", + "serde", + "serde_derive", + "toml_edit", +] + +[[package]] +name = "cargo-options" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad71bf996c8e5b9d28ef3472d7ee41f277edf4e38cd597f51ad0438d05d76ea" +dependencies = [ + "anstyle", + "clap 4.4.6", +] + +[[package]] +name = "cargo-platform" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-xwin" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afd4dab5fb38f9ec394ff7cd2aab95e5ddf581448afb9c42349ff849daed858" +dependencies = [ + "anyhow", + "cargo-config2", + "cargo-options", + "clap 4.4.6", + "dirs 5.0.1", + "fs-err", + "indicatif", + "paste", + "path-slash", + "rustls 0.21.10", + "rustls-pemfile 2.0.0", + "tracing-subscriber", + "ureq", + "which", + "xwin", +] + +[[package]] +name = "cargo-zigbuild" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187bb78a5f84e16e6c09df00e9b9e7b456ae13a156fbdde6a877f064f1e2c5dd" +dependencies = [ + "anyhow", + "cargo-options", + "cargo_metadata", + "clap 4.4.6", + "dirs 5.0.1", + "fs-err", + "path-slash", + "rustc_version", + "semver", + "serde", + "serde_json", + "shlex", + "target-lexicon", + "which", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cbindgen" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" +dependencies = [ + "heck 0.4.1", + "indexmap 1.9.3", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", + "tempfile", + "toml 0.5.11", +] + [[package]] name = "cc" version = "1.0.83" @@ -842,6 +999,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cfb" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b390793e912300f1aa713429f7fd0c391024e6c18b988962558bc4f96a349b1f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -854,6 +1022,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "charset" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e9079d1a12a2cc2bffb5db039c43661836ead4082120d5844f02555aca2d46" +dependencies = [ + "base64 0.13.1", + "encoding_rs", +] + [[package]] name = "chrono" version = "0.4.31" @@ -869,6 +1047,16 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "chumsky" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" +dependencies = [ + "hashbrown 0.14.3", + "stacker", +] + [[package]] name = "cipher" version = "0.4.4" @@ -919,6 +1107,47 @@ dependencies = [ "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97aeaa95557bd02f23fbb662f981670c3d20c5a26e69f7354b28f57092437fcd" +dependencies = [ + "clap 4.4.6", +] + +[[package]] +name = "clap_complete_command" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183495371ea78d4c9ff638bfc6497d46fed2396e4f9c50aebc1278a4a9919a3d" +dependencies = [ + "clap 4.4.6", + "clap_complete", + "clap_complete_fig", + "clap_complete_nushell", +] + +[[package]] +name = "clap_complete_fig" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e571d70e22ec91d34e1c5317c8308035a2280d925167646bf094fc5de1737c" +dependencies = [ + "clap 4.4.6", + "clap_complete", +] + +[[package]] +name = "clap_complete_nushell" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d02bc8b1a18ee47c4d2eec3fb5ac034dc68ebea6125b1509e9ccdffcddce66e" +dependencies = [ + "clap 4.4.6", + "clap_complete", +] + [[package]] name = "clap_derive" version = "3.2.25" @@ -941,7 +1170,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -959,6 +1188,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +[[package]] +name = "cli-table" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbb116d9e2c4be7011360d0c0bee565712c11e969c9609b25b619366dc379d" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "clircle" version = "0.3.0" @@ -1018,6 +1257,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "configparser" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec6d3da8e550377a85339063af6e3735f4b1d9392108da4e083a1b3b9820288" + [[package]] name = "console" version = "0.15.7" @@ -1039,23 +1284,21 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] @@ -1251,7 +1494,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1268,7 +1511,7 @@ checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1313,12 +1556,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + [[package]] name = "der" version = "0.6.1" @@ -1336,6 +1585,18 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "thiserror", + "zeroize", +] + [[package]] name = "digest" version = "0.10.7" @@ -1453,6 +1714,7 @@ dependencies = [ "aligned-vec", "dora-message", "eyre", + "maturin", "once_cell", "serde", "serde-with-expand-env", @@ -1903,7 +2165,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1927,23 +2189,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -2006,6 +2257,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fat-macho" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63fa117c7dcabeb8c83d5c229764cfa46518545d2dba5a9a08912014711f997b" +dependencies = [ + "goblin 0.7.1", +] + [[package]] name = "filetime" version = "0.2.22" @@ -2065,13 +2325,22 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2187,7 +2456,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -2257,7 +2526,7 @@ checksum = "ba330b70a5341d3bc730b8e205aaee97ddab5d9c448c4f51a7c2d924266fa8f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -2309,15 +2578,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2332,6 +2601,28 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "goblin" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" +dependencies = [ + "log", + "plain", + "scroll 0.11.0", +] + +[[package]] +name = "goblin" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb07a4ffed2093b118a525b1d8f5204ae274faed5604537caf7135d0f18d9887" +dependencies = [ + "log", + "plain", + "scroll 0.12.0", +] + [[package]] name = "grep-cli" version = "0.1.9" @@ -2386,9 +2677,13 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -2551,9 +2846,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2569,6 +2864,22 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ignore" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata 0.4.3", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indenter" version = "0.3.3" @@ -2592,7 +2903,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.3", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", ] [[package]] @@ -2747,7 +3072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", - "rustix 0.38.20", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -2761,10 +3086,28 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.9" +name = "itertools" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jni" @@ -2880,6 +3223,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lddtree" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88a93876d2485ede9c97d698c164cf5c024491908483964a998faae9705dea6" +dependencies = [ + "fs-err", + "glob", + "goblin 0.8.0", +] + [[package]] name = "lexical-core" version = "0.8.5" @@ -2946,9 +3300,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.148" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libgit2-sys" @@ -3022,9 +3376,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "local-ip-address" @@ -3057,6 +3411,12 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lzxd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784462f20dddd9dfdb45de963fa4ad4a288cb10a7889ac5d2c34fb6481c6b213" + [[package]] name = "macro_rules_attribute" version = "0.1.3" @@ -3073,6 +3433,17 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +[[package]] +name = "mailparse" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa" +dependencies = [ + "charset", + "data-encoding", + "quoted_printable", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3097,6 +3468,72 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maturin" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a36720938833f89c67b698065cf6707a95f6df954a2c7796097ddafe2ef45c5" +dependencies = [ + "anyhow", + "base64 0.21.4", + "bytesize", + "cargo-config2", + "cargo-options", + "cargo-xwin", + "cargo-zigbuild", + "cargo_metadata", + "cbindgen", + "cc", + "clap 4.4.6", + "clap_complete_command", + "configparser", + "console", + "dialoguer", + "dirs 5.0.1", + "dunce", + "fat-macho", + "flate2", + "fs-err", + "glob", + "goblin 0.7.1", + "ignore", + "indexmap 2.0.2", + "itertools 0.12.0", + "lddtree", + "minijinja", + "multipart", + "normpath", + "once_cell", + "path-slash", + "pep440_rs", + "pep508_rs", + "platform-info", + "pyproject-toml", + "python-pkginfo", + "regex", + "rustc_version", + "rustls 0.21.10", + "rustls-pemfile 2.0.0", + "semver", + "serde", + "serde_json", + "sha2", + "tar", + "target-lexicon", + "tempfile", + "textwrap", + "thiserror", + "time", + "toml 0.8.8", + "toml_edit", + "tracing", + "tracing-subscriber", + "ureq", + "url", + "wild", + "zip", +] + [[package]] name = "md5" version = "0.7.0" @@ -3151,6 +3588,25 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minijinja" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431d72874542d43aba1ca605870eacab134fdeb0c8fe27666ecf4b2662239df2" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3221,6 +3677,31 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "msi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226b2404f03d2cf47375b9715c8adfae4e388bb2377cff908e8a40f31e421514" +dependencies = [ + "byteorder", + "cfb", + "encoding_rs", + "uuid", +] + +[[package]] +name = "multipart" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" +dependencies = [ + "log", + "mime", + "mime_guess", + "rand", + "tempfile", +] + [[package]] name = "multiple-daemons-example-node" version = "0.3.0" @@ -3370,6 +3851,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "normpath" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec60c60a693226186f5d6edf073232bfb6464ed97eb22cf3b01c1e8198fd97f5" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "notify" version = "5.2.0" @@ -3531,6 +4021,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "nvml-wrapper" version = "0.9.0" @@ -3852,6 +4348,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "path_abs" version = "0.5.1" @@ -3870,11 +4372,39 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pep440_rs" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887f66cc62717ea72caac4f1eb4e6f392224da3ffff3f40ec13ab427802746d6" +dependencies = [ + "lazy_static", + "regex", + "serde", + "unicode-width", +] + +[[package]] +name = "pep508_rs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4516b53d9ea6112ebb38b4af08d5707d30b994fb7f98ff133c5dcf7ed8fa854" +dependencies = [ + "once_cell", + "pep440_rs", + "regex", + "serde", + "thiserror", + "tracing", + "unicode-width", + "url", +] + [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" @@ -3907,7 +4437,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -3948,7 +4478,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -4002,6 +4532,22 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "platform-info" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6259c4860e53bf665016f1b2f46a8859cadfa717581dc9d597ae4069de6300f" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "plist" version = "1.5.0" @@ -4121,6 +4667,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4169,9 +4721,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.68" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -4193,12 +4745,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "pyo3" version = "0.20.0" @@ -4247,7 +4808,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -4259,7 +4820,35 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", +] + +[[package]] +name = "pyproject-toml" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46d4a5e69187f23a29f8aa0ea57491d104ba541bc55f76552c2a74962aa20e04" +dependencies = [ + "indexmap 2.0.2", + "pep440_rs", + "pep508_rs", + "serde", + "toml 0.8.8", +] + +[[package]] +name = "python-pkginfo" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037469c164f08c891bf6d69ca02f1d56210011451e229618669777df82124cfa" +dependencies = [ + "flate2", + "fs-err", + "mailparse", + "rfc2047-decoder", + "tar", + "thiserror", + "zip", ] [[package]] @@ -4307,7 +4896,7 @@ checksum = "c956be1b23f4261676aed05a0046e204e8a6836e50203902683a718af0797989" dependencies = [ "bytes", "rand", - "ring", + "ring 0.16.20", "rustc-hash", "rustls 0.20.9", "rustls-native-certs", @@ -4333,13 +4922,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] +[[package]] +name = "quoted_printable" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" + [[package]] name = "radium" version = "0.7.0" @@ -4433,6 +5028,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -4476,6 +5080,17 @@ dependencies = [ "regex-syntax 0.7.5", ] +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4516,7 +5131,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.10", "serde", "serde_json", "serde_urlencoded", @@ -4530,6 +5145,20 @@ dependencies = [ "winreg", ] +[[package]] +name = "rfc2047-decoder" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fc4b4e52897c3e30b12b7e9b04461215b647fbe66f6def60dd8edbce14ec2e" +dependencies = [ + "base64 0.21.4", + "charset", + "chumsky", + "memchr", + "quoted_printable", + "thiserror", +] + [[package]] name = "rgb" version = "0.8.36" @@ -4549,11 +5178,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "ringbuffer-spsc" version = "0.1.9" @@ -4714,15 +5357,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.8", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.12", + "windows-sys 0.52.0", ] [[package]] @@ -4732,19 +5375,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring", + "ring 0.17.7", "rustls-webpki", "sct", ] @@ -4756,7 +5399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.3", "schannel", "security-framework", ] @@ -4770,14 +5413,30 @@ dependencies = [ "base64 0.21.4", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64 0.21.4", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.7", + "untrusted 0.9.0", ] [[package]] @@ -4859,14 +5518,54 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" +dependencies = [ + "scroll_derive 0.11.1", +] + +[[package]] +name = "scroll" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" +dependencies = [ + "scroll_derive 0.12.0", +] + +[[package]] +name = "scroll_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "scroll_derive" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "sct" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -4897,12 +5596,15 @@ name = "semver" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +dependencies = [ + "serde", +] [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -4937,13 +5639,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -4965,7 +5667,16 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", +] + +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", ] [[package]] @@ -5090,6 +5801,12 @@ dependencies = [ "dirs 5.0.1", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook" version = "0.3.17" @@ -5145,6 +5862,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "socket2" version = "0.4.9" @@ -5173,11 +5896,22 @@ checksum = "2a004c141c54615778c01a6722f6453fae7013e501b2b1f2dfe5684037174721" dependencies = [ "io-extras", "io-lifetimes 2.0.2", - "rustix 0.38.20", + "rustix 0.38.28", "uuid", "windows-sys 0.48.0", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi 0.3.9", +] + [[package]] name = "speedy" version = "0.8.6" @@ -5196,7 +5930,7 @@ checksum = "7d395866cb6778625150f77a430cc0af764ce0300f6a3d3413477785fa34b6c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -5224,6 +5958,19 @@ dependencies = [ "der", ] +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "libc", + "psm", + "winapi 0.3.9", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -5279,9 +6026,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -5367,12 +6114,36 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +dependencies = [ + "cfg-if 1.0.0", + "fastrand 2.0.1", + "redox_syscall 0.4.1", + "rustix 0.38.28", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" version = "1.3.0" @@ -5388,7 +6159,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "rustix 0.38.20", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -5397,6 +6168,11 @@ name = "textwrap" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] [[package]] name = "thiserror" @@ -5415,7 +6191,7 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -5539,7 +6315,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -5567,6 +6343,49 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.0.2", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.9.2" @@ -5633,6 +6452,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5646,7 +6466,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -5684,6 +6504,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -5694,12 +6524,15 @@ dependencies = [ "nu-ansi-term 0.46.0", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -5708,6 +6541,17 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if 1.0.0", + "rand", + "static_assertions", +] + [[package]] name = "typenum" version = "1.17.0" @@ -5735,6 +6579,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -5747,6 +6600,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.22" @@ -5795,6 +6654,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "unwind_safe" version = "0.1.0" @@ -5812,15 +6677,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ureq" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" +dependencies = [ + "base64 0.21.4", + "flate2", + "log", + "once_cell", + "rustls 0.21.10", + "rustls-webpki", + "serde", + "serde_json", + "socks", + "url", + "webpki-roots 0.25.3", +] + [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -5855,7 +6740,7 @@ checksum = "f7e1ba1f333bd65ce3c9f27de592fcbc256dafe3af2717f56d7c87761fbaccf4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -5912,6 +6797,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "versions" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73a36bc44e3039f51fbee93e39f41225f6b17b380eb70cc2aab942df06b34dd" +dependencies = [ + "itertools 0.11.0", + "nom", +] + [[package]] name = "waker-fn" version = "1.1.1" @@ -5964,7 +6859,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -5998,7 +6893,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6042,8 +6937,8 @@ version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -6055,16 +6950,23 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "which" -version = "4.4.2" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14" dependencies = [ "either", "home", "once_cell", - "rustix 0.38.20", + "rustix 0.38.28", + "windows-sys 0.48.0", ] [[package]] @@ -6189,6 +7091,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -6219,6 +7130,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -6231,6 +7157,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.34.0" @@ -6249,6 +7181,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.34.0" @@ -6267,6 +7205,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.34.0" @@ -6285,6 +7229,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.34.0" @@ -6303,6 +7253,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -6315,6 +7271,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.34.0" @@ -6333,6 +7295,21 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winnow" +version = "0.5.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7520bbdec7211caa7c4e682eb1fbe07abe20cee6756b6e00f537c82c11816aa" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -6394,6 +7371,51 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" +dependencies = [ + "libc", + "linux-raw-sys 0.4.12", + "rustix 0.38.28", +] + +[[package]] +name = "xwin" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43e0202f5457b48558096cb7b36d0e473f267551a89c82ed72d73b01dfd4007" +dependencies = [ + "anyhow", + "bytes", + "cab", + "camino", + "clap 4.4.6", + "cli-table", + "crossbeam-channel", + "flate2", + "indicatif", + "memchr", + "msi", + "parking_lot", + "rayon", + "regex", + "serde", + "serde_json", + "sha2", + "tempfile", + "toml 0.8.8", + "tracing", + "tracing-subscriber", + "twox-hash", + "ureq", + "versions", + "walkdir", + "zip", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -6580,7 +7602,7 @@ dependencies = [ "quinn", "rustls 0.20.9", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 1.0.3", "webpki", "zenoh-cfg-properties", "zenoh-config", @@ -6618,9 +7640,9 @@ dependencies = [ "async-trait", "futures", "log", - "rustls-pemfile", + "rustls-pemfile 1.0.3", "webpki", - "webpki-roots", + "webpki-roots 0.22.6", "zenoh-cfg-properties", "zenoh-config", "zenoh-core", @@ -6713,7 +7735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdf3eaea2095d2c13fefdae25aca813b3644fc15e1441e16a4398b5113033753" dependencies = [ "hex", - "itertools", + "itertools 0.10.5", "lazy_static", "rand", "serde", @@ -6792,8 +7814,42 @@ dependencies = [ "zenoh-sync", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "crossbeam-utils", + "flate2", + "time", +] diff --git a/binaries/coordinator/Cargo.toml b/binaries/coordinator/Cargo.toml index 50062d47..0692ce4f 100644 --- a/binaries/coordinator/Cargo.toml +++ b/binaries/coordinator/Cargo.toml @@ -27,7 +27,7 @@ dora-tracing = { workspace = true, optional = true } futures-concurrency = "7.1.0" zenoh = "0.7.0-rc" serde_json = "1.0.86" -which = "4.3.0" +which = "5.0.0" thiserror = "1.0.37" ctrlc = "3.2.5" clap = { version = "4.0.3", features = ["derive"] } diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index e2ca4fab..a5ba2491 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -39,4 +39,4 @@ bincode = "1.3.3" async-trait = "0.1.64" arrow-schema = { workspace = true } aligned-vec = "0.5.0" -which = "4.3.0" +which = "5.0.0" diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index c5961e1d..f4908815 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_yaml = "0.9.11" serde_bytes = "0.11.12" once_cell = "1.13.0" -which = "4.3.0" +which = "5.0.0" uuid = { version = "1.2.1", features = ["serde"] } dora-message = { workspace = true } tracing = "0.1" diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 83ce0dc5..05c78b22 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,4 +1,5 @@ -use eyre::{bail, eyre, Context}; +use eyre::{bail, eyre, Context, ContextCompat}; +use maturin::BuildOptions; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, ffi::OsStr, From afcfec81f1b1aeb4164c3ddc0e7a551cd728fcaa Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 17:08:15 +0100 Subject: [PATCH 033/104] Fix path by using ; instead of : --- Cargo.lock | 962 +---------------------- examples/python-dataflow/run.rs | 25 +- examples/python-operator-dataflow/run.rs | 25 +- examples/python-ros2-dataflow/run.rs | 25 +- libraries/core/src/lib.rs | 3 +- 5 files changed, 61 insertions(+), 979 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f7c0215..29149b0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,12 +60,6 @@ dependencies = [ "serde", ] -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -801,168 +795,18 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cab" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6b4de23c7d39c0631fd3cc952d87951c86c75a13812d7247cb7a896e7b3551" -dependencies = [ - "byteorder", - "flate2", - "lzxd", - "time", -] - [[package]] name = "cache-padded" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" -[[package]] -name = "camino" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" -dependencies = [ - "serde", -] - [[package]] name = "capnp" version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dca085c2c7d9d65ad749d450b19b551efaa8e3476a439bdca07aca8533097f3" -[[package]] -name = "cargo-config2" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90811f7e462b86622e20549c1c43bb3861d1bf011a08c2ceceaeb51fdfc27402" -dependencies = [ - "home", - "serde", - "serde_derive", - "toml_edit", -] - -[[package]] -name = "cargo-options" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad71bf996c8e5b9d28ef3472d7ee41f277edf4e38cd597f51ad0438d05d76ea" -dependencies = [ - "anstyle", - "clap 4.4.6", -] - -[[package]] -name = "cargo-platform" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo-xwin" -version = "0.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afd4dab5fb38f9ec394ff7cd2aab95e5ddf581448afb9c42349ff849daed858" -dependencies = [ - "anyhow", - "cargo-config2", - "cargo-options", - "clap 4.4.6", - "dirs 5.0.1", - "fs-err", - "indicatif", - "paste", - "path-slash", - "rustls 0.21.10", - "rustls-pemfile 2.0.0", - "tracing-subscriber", - "ureq", - "which", - "xwin", -] - -[[package]] -name = "cargo-zigbuild" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187bb78a5f84e16e6c09df00e9b9e7b456ae13a156fbdde6a877f064f1e2c5dd" -dependencies = [ - "anyhow", - "cargo-options", - "cargo_metadata", - "clap 4.4.6", - "dirs 5.0.1", - "fs-err", - "path-slash", - "rustc_version", - "semver", - "serde", - "serde_json", - "shlex", - "target-lexicon", - "which", -] - -[[package]] -name = "cargo_metadata" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "cbindgen" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" -dependencies = [ - "heck 0.4.1", - "indexmap 1.9.3", - "log", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn 1.0.109", - "tempfile", - "toml 0.5.11", -] - [[package]] name = "cc" version = "1.0.83" @@ -999,17 +843,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cfb" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b390793e912300f1aa713429f7fd0c391024e6c18b988962558bc4f96a349b1f" -dependencies = [ - "byteorder", - "fnv", - "uuid", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -1022,16 +855,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "charset" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e9079d1a12a2cc2bffb5db039c43661836ead4082120d5844f02555aca2d46" -dependencies = [ - "base64 0.13.1", - "encoding_rs", -] - [[package]] name = "chrono" version = "0.4.31" @@ -1047,16 +870,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "chumsky" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" -dependencies = [ - "hashbrown 0.14.3", - "stacker", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1107,47 +920,6 @@ dependencies = [ "terminal_size", ] -[[package]] -name = "clap_complete" -version = "4.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97aeaa95557bd02f23fbb662f981670c3d20c5a26e69f7354b28f57092437fcd" -dependencies = [ - "clap 4.4.6", -] - -[[package]] -name = "clap_complete_command" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183495371ea78d4c9ff638bfc6497d46fed2396e4f9c50aebc1278a4a9919a3d" -dependencies = [ - "clap 4.4.6", - "clap_complete", - "clap_complete_fig", - "clap_complete_nushell", -] - -[[package]] -name = "clap_complete_fig" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e571d70e22ec91d34e1c5317c8308035a2280d925167646bf094fc5de1737c" -dependencies = [ - "clap 4.4.6", - "clap_complete", -] - -[[package]] -name = "clap_complete_nushell" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d02bc8b1a18ee47c4d2eec3fb5ac034dc68ebea6125b1509e9ccdffcddce66e" -dependencies = [ - "clap 4.4.6", - "clap_complete", -] - [[package]] name = "clap_derive" version = "3.2.25" @@ -1188,16 +960,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" -[[package]] -name = "cli-table" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbb116d9e2c4be7011360d0c0bee565712c11e969c9609b25b619366dc379d" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "clircle" version = "0.3.0" @@ -1257,12 +1019,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "configparser" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec6d3da8e550377a85339063af6e3735f4b1d9392108da4e083a1b3b9820288" - [[package]] name = "console" version = "0.15.7" @@ -1562,12 +1318,6 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "data-encoding" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" - [[package]] name = "der" version = "0.6.1" @@ -1585,18 +1335,6 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "thiserror", - "zeroize", -] - [[package]] name = "digest" version = "0.10.7" @@ -1714,7 +1452,6 @@ dependencies = [ "aligned-vec", "dora-message", "eyre", - "maturin", "once_cell", "serde", "serde-with-expand-env", @@ -2257,15 +1994,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" -[[package]] -name = "fat-macho" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63fa117c7dcabeb8c83d5c229764cfa46518545d2dba5a9a08912014711f997b" -dependencies = [ - "goblin 0.7.1", -] - [[package]] name = "filetime" version = "0.2.22" @@ -2332,15 +2060,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-err" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" -dependencies = [ - "autocfg", -] - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2601,28 +2320,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "goblin" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" -dependencies = [ - "log", - "plain", - "scroll 0.11.0", -] - -[[package]] -name = "goblin" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb07a4ffed2093b118a525b1d8f5204ae274faed5604537caf7135d0f18d9887" -dependencies = [ - "log", - "plain", - "scroll 0.12.0", -] - [[package]] name = "grep-cli" version = "0.1.9" @@ -2680,10 +2377,6 @@ name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" -dependencies = [ - "ahash", - "allocator-api2", -] [[package]] name = "heck" @@ -2864,22 +2557,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "ignore" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata 0.4.3", - "same-file", - "walkdir", - "winapi-util", -] - [[package]] name = "indenter" version = "0.3.3" @@ -2904,20 +2581,6 @@ checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", "hashbrown 0.14.3", - "serde", -] - -[[package]] -name = "indicatif" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "unicode-width", ] [[package]] @@ -3085,24 +2748,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.9" @@ -3223,17 +2868,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" -[[package]] -name = "lddtree" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88a93876d2485ede9c97d698c164cf5c024491908483964a998faae9705dea6" -dependencies = [ - "fs-err", - "glob", - "goblin 0.8.0", -] - [[package]] name = "lexical-core" version = "0.8.5" @@ -3411,12 +3045,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lzxd" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784462f20dddd9dfdb45de963fa4ad4a288cb10a7889ac5d2c34fb6481c6b213" - [[package]] name = "macro_rules_attribute" version = "0.1.3" @@ -3433,17 +3061,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" -[[package]] -name = "mailparse" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa" -dependencies = [ - "charset", - "data-encoding", - "quoted_printable", -] - [[package]] name = "malloc_buf" version = "0.0.6" @@ -3468,72 +3085,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "maturin" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a36720938833f89c67b698065cf6707a95f6df954a2c7796097ddafe2ef45c5" -dependencies = [ - "anyhow", - "base64 0.21.4", - "bytesize", - "cargo-config2", - "cargo-options", - "cargo-xwin", - "cargo-zigbuild", - "cargo_metadata", - "cbindgen", - "cc", - "clap 4.4.6", - "clap_complete_command", - "configparser", - "console", - "dialoguer", - "dirs 5.0.1", - "dunce", - "fat-macho", - "flate2", - "fs-err", - "glob", - "goblin 0.7.1", - "ignore", - "indexmap 2.0.2", - "itertools 0.12.0", - "lddtree", - "minijinja", - "multipart", - "normpath", - "once_cell", - "path-slash", - "pep440_rs", - "pep508_rs", - "platform-info", - "pyproject-toml", - "python-pkginfo", - "regex", - "rustc_version", - "rustls 0.21.10", - "rustls-pemfile 2.0.0", - "semver", - "serde", - "serde_json", - "sha2", - "tar", - "target-lexicon", - "tempfile", - "textwrap", - "thiserror", - "time", - "toml 0.8.8", - "toml_edit", - "tracing", - "tracing-subscriber", - "ureq", - "url", - "wild", - "zip", -] - [[package]] name = "md5" version = "0.7.0" @@ -3588,25 +3139,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "minijinja" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431d72874542d43aba1ca605870eacab134fdeb0c8fe27666ecf4b2662239df2" -dependencies = [ - "serde", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3677,31 +3209,6 @@ dependencies = [ "ws2_32-sys", ] -[[package]] -name = "msi" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b2404f03d2cf47375b9715c8adfae4e388bb2377cff908e8a40f31e421514" -dependencies = [ - "byteorder", - "cfb", - "encoding_rs", - "uuid", -] - -[[package]] -name = "multipart" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" -dependencies = [ - "log", - "mime", - "mime_guess", - "rand", - "tempfile", -] - [[package]] name = "multiple-daemons-example-node" version = "0.3.0" @@ -3851,15 +3358,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "normpath" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec60c60a693226186f5d6edf073232bfb6464ed97eb22cf3b01c1e8198fd97f5" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "notify" version = "5.2.0" @@ -4021,12 +3519,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "nvml-wrapper" version = "0.9.0" @@ -4348,12 +3840,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "path-slash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" - [[package]] name = "path_abs" version = "0.5.1" @@ -4372,34 +3858,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "pep440_rs" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "887f66cc62717ea72caac4f1eb4e6f392224da3ffff3f40ec13ab427802746d6" -dependencies = [ - "lazy_static", - "regex", - "serde", - "unicode-width", -] - -[[package]] -name = "pep508_rs" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4516b53d9ea6112ebb38b4af08d5707d30b994fb7f98ff133c5dcf7ed8fa854" -dependencies = [ - "once_cell", - "pep440_rs", - "regex", - "serde", - "thiserror", - "tracing", - "unicode-width", - "url", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -4532,22 +3990,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - -[[package]] -name = "platform-info" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6259c4860e53bf665016f1b2f46a8859cadfa717581dc9d597ae4069de6300f" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "plist" version = "1.5.0" @@ -4667,12 +4109,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4745,21 +4181,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 1.0.109", ] -[[package]] -name = "psm" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" -dependencies = [ - "cc", -] - [[package]] name = "pyo3" version = "0.20.0" @@ -4823,34 +4250,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "pyproject-toml" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d4a5e69187f23a29f8aa0ea57491d104ba541bc55f76552c2a74962aa20e04" -dependencies = [ - "indexmap 2.0.2", - "pep440_rs", - "pep508_rs", - "serde", - "toml 0.8.8", -] - -[[package]] -name = "python-pkginfo" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037469c164f08c891bf6d69ca02f1d56210011451e229618669777df82124cfa" -dependencies = [ - "flate2", - "fs-err", - "mailparse", - "rfc2047-decoder", - "tar", - "thiserror", - "zip", -] - [[package]] name = "pythonize" version = "0.20.0" @@ -4929,12 +4328,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "quoted_printable" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" - [[package]] name = "radium" version = "0.7.0" @@ -5028,15 +4421,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_users" version = "0.4.3" @@ -5145,20 +4529,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rfc2047-decoder" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fc4b4e52897c3e30b12b7e9b04461215b647fbe66f6def60dd8edbce14ec2e" -dependencies = [ - "base64 0.21.4", - "charset", - "chumsky", - "memchr", - "quoted_printable", - "thiserror", -] - [[package]] name = "rgb" version = "0.8.36" @@ -5399,7 +4769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.3", + "rustls-pemfile", "schannel", "security-framework", ] @@ -5413,22 +4783,6 @@ dependencies = [ "base64 0.21.4", ] -[[package]] -name = "rustls-pemfile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" -dependencies = [ - "base64 0.21.4", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5518,46 +4872,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" -[[package]] -name = "scroll" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" -dependencies = [ - "scroll_derive 0.11.1", -] - -[[package]] -name = "scroll" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" -dependencies = [ - "scroll_derive 0.12.0", -] - -[[package]] -name = "scroll_derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "scroll_derive" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "sct" version = "0.7.0" @@ -5596,9 +4910,6 @@ name = "semver" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" -dependencies = [ - "serde", -] [[package]] name = "serde" @@ -5670,15 +4981,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "serde_spanned" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5801,12 +5103,6 @@ dependencies = [ "dirs 5.0.1", ] -[[package]] -name = "shlex" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" - [[package]] name = "signal-hook" version = "0.3.17" @@ -5862,12 +5158,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" -[[package]] -name = "smawk" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" - [[package]] name = "socket2" version = "0.4.9" @@ -5901,17 +5191,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "socks" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" -dependencies = [ - "byteorder", - "libc", - "winapi 0.3.9", -] - [[package]] name = "speedy" version = "0.8.6" @@ -5958,19 +5237,6 @@ dependencies = [ "der", ] -[[package]] -name = "stacker" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" -dependencies = [ - "cc", - "cfg-if 1.0.0", - "libc", - "psm", - "winapi 0.3.9", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -6114,36 +5380,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "tar" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "target-lexicon" version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" -[[package]] -name = "tempfile" -version = "3.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" -dependencies = [ - "cfg-if 1.0.0", - "fastrand 2.0.1", - "redox_syscall 0.4.1", - "rustix 0.38.28", - "windows-sys 0.52.0", -] - [[package]] name = "termcolor" version = "1.3.0" @@ -6168,11 +5410,6 @@ name = "textwrap" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" -dependencies = [ - "smawk", - "unicode-linebreak", - "unicode-width", -] [[package]] name = "thiserror" @@ -6343,49 +5580,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" -dependencies = [ - "indexmap 2.0.2", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tonic" version = "0.9.2" @@ -6452,7 +5646,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6504,16 +5697,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -6524,15 +5707,12 @@ dependencies = [ "nu-ansi-term 0.46.0", "once_cell", "regex", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", - "tracing-serde", ] [[package]] @@ -6541,17 +5721,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if 1.0.0", - "rand", - "static_assertions", -] - [[package]] name = "typenum" version = "1.17.0" @@ -6579,15 +5748,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.13" @@ -6600,12 +5760,6 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-normalization" version = "0.1.22" @@ -6677,25 +5831,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "ureq" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" -dependencies = [ - "base64 0.21.4", - "flate2", - "log", - "once_cell", - "rustls 0.21.10", - "rustls-webpki", - "serde", - "serde_json", - "socks", - "url", - "webpki-roots 0.25.3", -] - [[package]] name = "url" version = "2.5.0" @@ -6705,7 +5840,6 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] [[package]] @@ -6797,16 +5931,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "versions" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c73a36bc44e3039f51fbee93e39f41225f6b17b380eb70cc2aab942df06b34dd" -dependencies = [ - "itertools 0.11.0", - "nom", -] - [[package]] name = "waker-fn" version = "1.1.1" @@ -6950,12 +6074,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "webpki-roots" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" - [[package]] name = "which" version = "5.0.0" @@ -7301,15 +6419,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" -[[package]] -name = "winnow" -version = "0.5.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7520bbdec7211caa7c4e682eb1fbe07abe20cee6756b6e00f537c82c11816aa" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" @@ -7371,51 +6480,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xattr" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" -dependencies = [ - "libc", - "linux-raw-sys 0.4.12", - "rustix 0.38.28", -] - -[[package]] -name = "xwin" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43e0202f5457b48558096cb7b36d0e473f267551a89c82ed72d73b01dfd4007" -dependencies = [ - "anyhow", - "bytes", - "cab", - "camino", - "clap 4.4.6", - "cli-table", - "crossbeam-channel", - "flate2", - "indicatif", - "memchr", - "msi", - "parking_lot", - "rayon", - "regex", - "serde", - "serde_json", - "sha2", - "tempfile", - "toml 0.8.8", - "tracing", - "tracing-subscriber", - "twox-hash", - "ureq", - "versions", - "walkdir", - "zip", -] - [[package]] name = "yaml-rust" version = "0.4.5" @@ -7602,7 +6666,7 @@ dependencies = [ "quinn", "rustls 0.20.9", "rustls-native-certs", - "rustls-pemfile 1.0.3", + "rustls-pemfile", "webpki", "zenoh-cfg-properties", "zenoh-config", @@ -7640,9 +6704,9 @@ dependencies = [ "async-trait", "futures", "log", - "rustls-pemfile 1.0.3", + "rustls-pemfile", "webpki", - "webpki-roots 0.22.6", + "webpki-roots", "zenoh-cfg-properties", "zenoh-config", "zenoh-core", @@ -7735,7 +6799,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdf3eaea2095d2c13fefdae25aca813b3644fc15e1441e16a4398b5113033753" dependencies = [ "hex", - "itertools 0.10.5", + "itertools", "lazy_static", "rand", "serde", @@ -7839,17 +6903,3 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" - -[[package]] -name = "zip" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" -dependencies = [ - "byteorder", - "bzip2", - "crc32fast", - "crossbeam-utils", - "flate2", - "time", -] diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 16e9d2d6..c8b062d1 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -31,13 +31,24 @@ async fn main() -> eyre::Result<()> { } else { venv.join("bin") }; - std::env::set_var( - "PATH", - format!( - "{}:{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); + + if cfg!(windows) { + std::env::set_var( + "PATH", + format!( + "{};{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } else { + std::env::set_var( + "PATH", + format!( + "{}:{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } run( get_pip_path().context("Could not get pip binary")?, diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index 4cb19c6d..5451e0c4 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -31,13 +31,24 @@ async fn main() -> eyre::Result<()> { } else { venv.join("bin") }; - std::env::set_var( - "PATH", - format!( - "{}:{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); + + if cfg!(windows) { + std::env::set_var( + "PATH", + format!( + "{};{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } else { + std::env::set_var( + "PATH", + format!( + "{}:{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } run( get_pip_path().context("Could not get pip binary")?, diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index f136ae46..642ded84 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -31,13 +31,24 @@ async fn main() -> eyre::Result<()> { } else { venv.join("bin") }; - std::env::set_var( - "PATH", - format!( - "{}:{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); + + if cfg!(windows) { + std::env::set_var( + "PATH", + format!( + "{};{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } else { + std::env::set_var( + "PATH", + format!( + "{}:{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); + } run( get_pip_path().context("Could not get pip binary")?, diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 05c78b22..83ce0dc5 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,5 +1,4 @@ -use eyre::{bail, eyre, Context, ContextCompat}; -use maturin::BuildOptions; +use eyre::{bail, eyre, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, ffi::OsStr, From ed83fe05c1a8891d46dd3eaba032051f772a529b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 18:12:02 +0100 Subject: [PATCH 034/104] Fix pip upgrade --- examples/python-dataflow/run.rs | 4 ++-- examples/python-operator-dataflow/run.rs | 4 ++-- examples/python-ros2-dataflow/run.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index c8b062d1..62fb7812 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -51,8 +51,8 @@ async fn main() -> eyre::Result<()> { } run( - get_pip_path().context("Could not get pip binary")?, - &["install", "--upgrade", "pip"], + get_python_path().context("Could not get pip binary")?, + &["-m", "pip", "install", "--upgrade", "pip"], None, ) .await diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index 5451e0c4..ab40f6b0 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -51,8 +51,8 @@ async fn main() -> eyre::Result<()> { } run( - get_pip_path().context("Could not get pip binary")?, - &["install", "--upgrade", "pip"], + get_python_path().context("Could not get pip binary")?, + &["-m", "pip", "install", "--upgrade", "pip"], None, ) .await diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 642ded84..65e14129 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -51,8 +51,8 @@ async fn main() -> eyre::Result<()> { } run( - get_pip_path().context("Could not get pip binary")?, - &["install", "--upgrade", "pip"], + get_python_path().context("Could not get pip binary")?, + &["-m", "pip", "install", "--upgrade", "pip"], None, ) .await From 3ad740a65c043708efaddc25a080a046d3655daf Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Jan 2024 20:41:37 +0100 Subject: [PATCH 035/104] bump to latest version of torch --- examples/python-dataflow/requirements.txt | 6 ++---- examples/python-operator-dataflow/requirements.txt | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/python-dataflow/requirements.txt b/examples/python-dataflow/requirements.txt index 4fa1e7d0..a8f02b0a 100644 --- a/examples/python-dataflow/requirements.txt +++ b/examples/python-dataflow/requirements.txt @@ -14,10 +14,8 @@ PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 thop>=0.1.1 # FLOPs computation ---extra-index-url https://download.pytorch.org/whl/cpu -torch>=1.7.0 # see https://pytorch.org/get-started/locally (recommended) ---extra-index-url https://download.pytorch.org/whl/cpu -torchvision>=0.8.1 +torch # see https://pytorch.org/get-started/locally (recommended) +torchvision tqdm>=4.64.0 # Logging ------------------------------------- diff --git a/examples/python-operator-dataflow/requirements.txt b/examples/python-operator-dataflow/requirements.txt index 11dc59cd..78d804d7 100644 --- a/examples/python-operator-dataflow/requirements.txt +++ b/examples/python-operator-dataflow/requirements.txt @@ -10,10 +10,8 @@ Pillow>=7.1.2 PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 ---extra-index-url https://download.pytorch.org/whl/cpu -torch>=1.7.0 ---extra-index-url https://download.pytorch.org/whl/cpu -torchvision>=0.8.1 +torch # see https://pytorch.org/get-started/locally (recommended) +torchvision tqdm>=4.64.0 protobuf<=3.20.1 # https://github.com/ultralytics/yolov5/issues/8012 From 5a187f5ba90f084593d64f720cae3f5585d13940 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 10:52:37 +0100 Subject: [PATCH 036/104] Reload only if on windows --- examples/python-dataflow/object_detection.py | 10 ++++++---- examples/python-dataflow/plot.py | 1 - examples/python-operator-dataflow/object_detection.py | 9 ++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py index 2107b30f..5cb9b7e0 100755 --- a/examples/python-dataflow/object_detection.py +++ b/examples/python-dataflow/object_detection.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Callable -from dora import Node - +import os import cv2 import numpy as np import torch -model = torch.hub.load("ultralytics/yolov5", "yolov5n") +from dora import Node + +# Reload only if on Windows +reload = os.name == "nt" +model = torch.hub.load("ultralytics/yolov5", "yolov5n", force_reload=reload) node = Node() diff --git a/examples/python-dataflow/plot.py b/examples/python-dataflow/plot.py index 652f25a7..035fc41d 100755 --- a/examples/python-dataflow/plot.py +++ b/examples/python-dataflow/plot.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import os -from typing import Callable from dora import Node from dora import DoraStatus diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index 4c2a7fed..8c49a307 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -2,11 +2,10 @@ # -*- coding: utf-8 -*- -import cv2 import numpy as np import pyarrow as pa import torch - +import os from dora import DoraStatus pa.array([]) @@ -21,7 +20,11 @@ class Operator: """ def __init__(self): - self.model = torch.hub.load("ultralytics/yolov5", "yolov5n") + # Reload only if on Windows + reload = os.name == "nt" + self.model = torch.hub.load( + "ultralytics/yolov5", "yolov5n", force_reload=reload + ) def on_event( self, From ab5d918ffd7cfabd528c58eec22674c8017f54f1 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 11:58:44 +0100 Subject: [PATCH 037/104] Make initialisation message error more straightforward --- binaries/daemon/src/pending.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/binaries/daemon/src/pending.rs b/binaries/daemon/src/pending.rs index 834d668f..918cf6c4 100644 --- a/binaries/daemon/src/pending.rs +++ b/binaries/daemon/src/pending.rs @@ -1,4 +1,7 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + intrinsics::mir::Len, +}; use dora_core::{ config::NodeId, @@ -6,7 +9,7 @@ use dora_core::{ daemon_messages::{DaemonReply, DataflowId, Timestamped}, message::uhlc::{Timestamp, HLC}, }; -use eyre::{bail, Context}; +use eyre::{bail, Context, ContextCompat}; use tokio::{net::TcpStream, sync::oneshot}; use crate::tcp_utils::tcp_send; @@ -128,11 +131,20 @@ impl PendingNodes { None => Ok(()), } } else { + let node_id_message = if self.exited_before_subscribe.len() == 1 { + self.exited_before_subscribe + .iter() + .next() + .map(|node_id| node_id.to_string()) + .unwrap_or("".to_string()) + } else { + "".to_string() + }; Err(format!( "Some nodes exited before subscribing to dora: {:?}\n\n\ This is typically happens when an initialization error occurs in the node or operator. To check the output of the failed - nodes, run `dora logs {} `.", + nodes, run `dora logs {} {node_id_message}`.", self.exited_before_subscribe, self.dataflow_id )) }; From 3698ca08675abc50057625d95dea276479652d86 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 12:07:52 +0100 Subject: [PATCH 038/104] Remove dora-record from operator dataflow --- examples/python-dataflow/object_detection.py | 3 +-- examples/python-operator-dataflow/dataflow.yml | 7 ------- examples/python-operator-dataflow/object_detection.py | 6 +----- examples/python-operator-dataflow/requirements.txt | 9 ++++----- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py index 5cb9b7e0..b9fff333 100755 --- a/examples/python-dataflow/object_detection.py +++ b/examples/python-dataflow/object_detection.py @@ -9,8 +9,7 @@ import torch from dora import Node # Reload only if on Windows -reload = os.name == "nt" -model = torch.hub.load("ultralytics/yolov5", "yolov5n", force_reload=reload) +model = torch.hub.load("ultralytics/yolov5", "yolov5n") node = Node() diff --git a/examples/python-operator-dataflow/dataflow.yml b/examples/python-operator-dataflow/dataflow.yml index 72630d27..92bf5f2b 100644 --- a/examples/python-operator-dataflow/dataflow.yml +++ b/examples/python-operator-dataflow/dataflow.yml @@ -21,10 +21,3 @@ nodes: inputs: image: webcam/image bbox: object_detection/bbox - - id: dora-record - custom: - build: cargo build -p dora-record - source: ../../target/debug/dora-record - inputs: - image: webcam/image - bbox: object_detection/bbox diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index 8c49a307..90dbf449 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -20,11 +20,7 @@ class Operator: """ def __init__(self): - # Reload only if on Windows - reload = os.name == "nt" - self.model = torch.hub.load( - "ultralytics/yolov5", "yolov5n", force_reload=reload - ) + self.model = torch.hub.load("ultralytics/yolov5", "yolov5n") def on_event( self, diff --git a/examples/python-operator-dataflow/requirements.txt b/examples/python-operator-dataflow/requirements.txt index 78d804d7..a8f02b0a 100644 --- a/examples/python-operator-dataflow/requirements.txt +++ b/examples/python-operator-dataflow/requirements.txt @@ -3,17 +3,20 @@ # Base ---------------------------------------- ultralytics +gitpython +ipython # interactive notebook matplotlib>=3.2.2 numpy>=1.18.5 opencv-python>=4.1.1 Pillow>=7.1.2 +psutil # system resources PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 +thop>=0.1.1 # FLOPs computation torch # see https://pytorch.org/get-started/locally (recommended) torchvision tqdm>=4.64.0 -protobuf<=3.20.1 # https://github.com/ultralytics/yolov5/issues/8012 # Logging ------------------------------------- tensorboard>=2.4.1 @@ -36,13 +39,9 @@ seaborn>=0.11.0 # openvino-dev # OpenVINO export # Extras -------------------------------------- -ipython # interactive notebook -psutil # system utilization -thop>=0.1.1 # FLOPs computation # albumentations>=1.0.3 # pycocotools>=2.0 # COCO mAP # roboflow opencv-python>=4.1.1 -pyarrow maturin \ No newline at end of file From fdda91d9156cee3158638fb2abc73b41990c3c23 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 13:16:28 +0100 Subject: [PATCH 039/104] Fix mistakenly imported Len --- binaries/daemon/src/pending.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/binaries/daemon/src/pending.rs b/binaries/daemon/src/pending.rs index 918cf6c4..9ba907ff 100644 --- a/binaries/daemon/src/pending.rs +++ b/binaries/daemon/src/pending.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{HashMap, HashSet}, - intrinsics::mir::Len, -}; +use std::collections::{HashMap, HashSet}; use dora_core::{ config::NodeId, @@ -9,7 +6,7 @@ use dora_core::{ daemon_messages::{DaemonReply, DataflowId, Timestamped}, message::uhlc::{Timestamp, HLC}, }; -use eyre::{bail, Context, ContextCompat}; +use eyre::{bail, Context}; use tokio::{net::TcpStream, sync::oneshot}; use crate::tcp_utils::tcp_send; From f3cd7725ed649de6cf0342e6e4681537efb0846a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 14:47:19 +0100 Subject: [PATCH 040/104] Replace yolov5 with yolov8 --- .gitignore | 1 + examples/python-dataflow/object_detection.py | 16 ++++++++++------ .../python-operator-dataflow/object_detection.py | 14 ++++++++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 557fa2e1..59491033 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ # Remove arrow file from dora-record **/*.arrow +*.pt # Removing images. *.jpg diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py index b9fff333..5b4788d8 100755 --- a/examples/python-dataflow/object_detection.py +++ b/examples/python-dataflow/object_detection.py @@ -1,15 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import os import cv2 import numpy as np -import torch +from ultralytics import YOLO from dora import Node +import pyarrow as pa -# Reload only if on Windows -model = torch.hub.load("ultralytics/yolov5", "yolov5n") +model = YOLO("yolov8n.pt") node = Node() @@ -23,9 +22,14 @@ for event in node: frame = cv2.imdecode(frame, -1) frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) results = model(frame) # includes NMS - arrays = np.array(results.xyxy[0].cpu()).tobytes() + # Process results + boxes = np.array(results[0].boxes.xyxy.cpu()) + conf = np.array(results[0].boxes.conf) + label = np.array(results[0].boxes.cls) + # concatenate them together + arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) - node.send_output("bbox", arrays, event["metadata"]) + node.send_output("bbox", pa.array(arrays.ravel()), event["metadata"]) else: print("[object detection] ignoring unexpected input:", event_id) elif event_type == "STOP": diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index 90dbf449..d1aed848 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -4,9 +4,9 @@ import numpy as np import pyarrow as pa -import torch -import os + from dora import DoraStatus +from ultralytics import YOLO pa.array([]) @@ -20,7 +20,7 @@ class Operator: """ def __init__(self): - self.model = torch.hub.load("ultralytics/yolov5", "yolov5n") + self.model = YOLO("yolov8n.pt") def on_event( self, @@ -50,6 +50,12 @@ class Operator: frame = dora_input["value"].to_numpy().reshape((CAMERA_HEIGHT, CAMERA_WIDTH, 3)) frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) results = self.model(frame) # includes NMS - arrays = pa.array(np.array(results.xyxy[0].cpu()).ravel()) + # Process results + boxes = np.array(results[0].boxes.xyxy.cpu()) + conf = np.array(results[0].boxes.conf) + label = np.array(results[0].boxes.cls) + # concatenate them together + arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) + send_output("bbox", arrays, dora_input["metadata"]) return DoraStatus.CONTINUE From 0f667375bae7b3b49f0733447c56b5762ea6e1b0 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 15:07:43 +0100 Subject: [PATCH 041/104] Convert numpy to pyarrow --- examples/python-operator-dataflow/object_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index d1aed848..d0c5b56d 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -57,5 +57,5 @@ class Operator: # concatenate them together arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) - send_output("bbox", arrays, dora_input["metadata"]) + send_output("bbox", pa.array(arrays), dora_input["metadata"]) return DoraStatus.CONTINUE From b63396b42e860472b1a7f1358a39f462c03b9450 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 15:39:30 +0100 Subject: [PATCH 042/104] ravel array bbox --- examples/python-operator-dataflow/object_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index d0c5b56d..d288986a 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -57,5 +57,5 @@ class Operator: # concatenate them together arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) - send_output("bbox", pa.array(arrays), dora_input["metadata"]) + send_output("bbox", pa.array(arrays.ravel()), dora_input["metadata"]) return DoraStatus.CONTINUE From 8b7cf8ebb944317a007c9cfca8a467d03e3f54a2 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 15:55:09 +0100 Subject: [PATCH 043/104] convert to cpu object detection output --- examples/python-dataflow/object_detection.py | 4 ++-- examples/python-operator-dataflow/object_detection.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py index 5b4788d8..70a0e712 100755 --- a/examples/python-dataflow/object_detection.py +++ b/examples/python-dataflow/object_detection.py @@ -24,8 +24,8 @@ for event in node: results = model(frame) # includes NMS # Process results boxes = np.array(results[0].boxes.xyxy.cpu()) - conf = np.array(results[0].boxes.conf) - label = np.array(results[0].boxes.cls) + conf = np.array(results[0].boxes.conf.cpu()) + label = np.array(results[0].boxes.cls.cpu()) # concatenate them together arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) diff --git a/examples/python-operator-dataflow/object_detection.py b/examples/python-operator-dataflow/object_detection.py index d288986a..a6bd9caf 100755 --- a/examples/python-operator-dataflow/object_detection.py +++ b/examples/python-operator-dataflow/object_detection.py @@ -52,8 +52,8 @@ class Operator: results = self.model(frame) # includes NMS # Process results boxes = np.array(results[0].boxes.xyxy.cpu()) - conf = np.array(results[0].boxes.conf) - label = np.array(results[0].boxes.cls) + conf = np.array(results[0].boxes.conf.cpu()) + label = np.array(results[0].boxes.cls.cpu()) # concatenate them together arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) From c82c853d027eb939fe5642c18344259731d6394b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 18:37:44 +0100 Subject: [PATCH 044/104] Download weights inside of dora to bypass yolov8 windows error Being able to download a file within a yaml declaration can later be integrated as a yaml feature. See: https://github.com/ultralytics/ultralytics/pull/7432#issuecomment-1883383862 --- Cargo.lock | 37 +++++++++++++++++++++++- Cargo.toml | 1 + examples/python-dataflow/run.rs | 7 +++++ examples/python-operator-dataflow/run.rs | 7 +++++ libraries/extensions/download/Cargo.toml | 2 +- 5 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29149b0c..dae262e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1511,6 +1511,7 @@ dependencies = [ "dora-coordinator", "dora-core", "dora-daemon", + "dora-download", "dora-tracing", "dunce", "eyre", @@ -2496,6 +2497,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls 0.21.10", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -4508,6 +4523,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "ipnet", "js-sys", "log", @@ -4516,16 +4532,19 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.21.10", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots 0.25.3", "winreg", ] @@ -5555,6 +5574,16 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.10", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -6074,6 +6103,12 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "which" version = "5.0.0" @@ -6706,7 +6741,7 @@ dependencies = [ "log", "rustls-pemfile", "webpki", - "webpki-roots", + "webpki-roots 0.22.6", "zenoh-cfg-properties", "zenoh-config", "zenoh-core", diff --git a/Cargo.toml b/Cargo.toml index 15a5334b..0e2e509f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,7 @@ dora-daemon = { workspace = true } dora-coordinator = { workspace = true } dora-core = { workspace = true } dora-tracing = { workspace = true } +dora-download = { workspace = true } dunce = "1.0.2" serde_yaml = "0.8.23" uuid = { version = "1.2.1", features = ["v4", "serde"] } diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 62fb7812..e194e062 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,4 +1,5 @@ use dora_core::{get_pip_path, get_python_path, run}; +use dora_download::download_file; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -72,6 +73,12 @@ async fn main() -> eyre::Result<()> { ) .await .context("maturin develop failed")?; + download_file( + "https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt", + Path::new("yolov8n.pt"), + ) + .await + .context("Could not download weights.")?; let dataflow = Path::new("dataflow.yml"); dora_daemon::Daemon::run_dataflow(dataflow).await?; diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index ab40f6b0..1413042f 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,4 +1,5 @@ use dora_core::{get_pip_path, get_python_path, run}; +use dora_download::download_file; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -73,6 +74,12 @@ async fn main() -> eyre::Result<()> { .await .context("maturin develop failed")?; + download_file( + "https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt", + Path::new("yolov8n.pt"), + ) + .await + .context("Could not download weights.")?; let dataflow = Path::new("dataflow.yml"); dora_daemon::Daemon::run_dataflow(dataflow).await?; diff --git a/libraries/extensions/download/Cargo.toml b/libraries/extensions/download/Cargo.toml index 266bde6a..b0885803 100644 --- a/libraries/extensions/download/Cargo.toml +++ b/libraries/extensions/download/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true [dependencies] eyre = "0.6.8" reqwest = { version = "0.11.12", default-features = false, features = [ - "rustls", + "rustls-tls", ] } tokio = { version = "1.24.2", features = ["fs"] } tracing = "0.1.36" From 6c6d1c643e745b1e335f83a87af7123c47dc9a8c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 20:27:47 +0100 Subject: [PATCH 045/104] bump to version v0.3.1-rc5 to test CD --- Cargo.lock | 68 +++++++++++++++---------------- Cargo.toml | 40 +++++++++--------- README.md | 12 +++--- apis/python/node/dora/__init__.py | 2 +- 4 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d43f9f8f..518a1d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -1013,7 +1013,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "flume", "zenoh", @@ -1021,7 +1021,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.0" +version = "0.3.1-rc5" [[package]] name = "concurrent-queue" @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "arrow", "eyre", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1460,7 +1460,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "aligned-vec", "dora-message", @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "aligned-vec", "arrow-schema", @@ -1508,7 +1508,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "eyre", "reqwest", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "arrow-data", "arrow-schema", @@ -1550,7 +1550,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1561,7 +1561,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "aligned-vec", "arrow", @@ -1590,7 +1590,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "arrow-array", "dora-node-api", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "cxx", "cxx-build", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "arrow", "dora-node-api", @@ -1628,7 +1628,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1637,14 +1637,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "cxx", "cxx-build", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "proc-macro2", "quote", @@ -1662,7 +1662,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "aligned-vec", "arrow", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "arrow", "dora-arrow-convert", @@ -1685,7 +1685,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "chrono", "dora-node-api", @@ -1758,7 +1758,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "aligned-vec", "arrow", @@ -1790,7 +1790,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -3280,14 +3280,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -3521,7 +3521,7 @@ checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -4688,7 +4688,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -4699,14 +4699,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "eyre", @@ -4714,7 +4714,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5118,7 +5118,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.0" +version = "0.3.1-rc5" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 0e2e509f..e1557eb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.0" +version = "0.3.1-rc5" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.0", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.0", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.0", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.0", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.0", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.0", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.0", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.0", path = "apis/c/node" } -dora-core = { version = "0.3.0", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.0", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.0", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.0", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.0", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.0", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.0", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.0", path = "libraries/message" } -dora-runtime = { version = "0.3.0", path = "binaries/runtime" } -dora-daemon = { version = "0.3.0", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.0", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1-rc5", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1-rc5", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1-rc5", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1-rc5", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1-rc5", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1-rc5", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1-rc5", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1-rc5", path = "apis/c/node" } +dora-core = { version = "0.3.1-rc5", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1-rc5", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1-rc5", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1-rc5", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1-rc5", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1-rc5", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1-rc5", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1-rc5", path = "libraries/message" } +dora-runtime = { version = "0.3.1-rc5", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1-rc5", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1-rc5", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/README.md b/README.md index 78f1f4ea..c1cb9b30 100644 --- a/README.md +++ b/README.md @@ -71,17 +71,17 @@ For more installation guideline, check out our installation guide here: https:// 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.0/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index aad31782..27aea294 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.0" +__version__ = "0.3.1-rc5" class DoraStatus(Enum): From c33d941f1a993734bd04b14111e6d19d7aa12e43 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 20:31:35 +0100 Subject: [PATCH 046/104] Add changelog --- Changelog.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Changelog.md b/Changelog.md index b3d5a411..d0e758ab 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,31 @@ # Changelog +## v0.3.1 (2024-01-09) + +## Features + +- Support legacy python by @haixuanTao in https://github.com/dora-rs/dora/pull/382 +- Add an error catch in python `on_event` when using hot-reloading by @haixuanTao in https://github.com/dora-rs/dora/pull/372 +- add cmake example by @XxChang in https://github.com/dora-rs/dora/pull/381 +- Bump opentelemetry metrics to 0.21 by @haixuanTao in https://github.com/dora-rs/dora/pull/383 +- Trace send_output as it can be a big source of overhead for large messages by @haixuanTao in https://github.com/dora-rs/dora/pull/384 +- Adding a timeout method to not block indefinitely next event by @haixuanTao in https://github.com/dora-rs/dora/pull/386 +- Adding `Vec` conversion by @haixuanTao in https://github.com/dora-rs/dora/pull/387 +- Dora cli renaming by @haixuanTao in https://github.com/dora-rs/dora/pull/399 +- Update `ros2-client` and `rustdds` dependencies to latest fork version by @phil-opp in https://github.com/dora-rs/dora/pull/397 + +## Fix + +- Fix window path error by @haixuanTao in https://github.com/dora-rs/dora/pull/398 +- Fix read error in C++ node input by @haixuanTao in https://github.com/dora-rs/dora/pull/406 +- Bump unsafe-libyaml from 0.2.9 to 0.2.10 by @dependabot in https://github.com/dora-rs/dora/pull/400 + +## New Contributors + +- @XxChang made their first contribution in https://github.com/dora-rs/dora/pull/381 + +**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.0...v0.3.1-rc5 + ## v0.3.0 (2023-11-01) ## Features From ee780df1450ec9815e82ecca78616a5648aa2ca8 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 20:41:56 +0100 Subject: [PATCH 047/104] Bump rust toolchain to use `stdsimd` feature on 1.72. See: https://github.com/tkaitchuck/aHash/issues/195 Related to: https://github.com/dora-rs/dora/actions/runs/7465955428/job/20316250830 --- examples/cmake-dataflow/DoraTargets.cmake | 2 +- rust-toolchain.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cmake-dataflow/DoraTargets.cmake b/examples/cmake-dataflow/DoraTargets.cmake index 6e9cb970..5fd45165 100644 --- a/examples/cmake-dataflow/DoraTargets.cmake +++ b/examples/cmake-dataflow/DoraTargets.cmake @@ -15,7 +15,7 @@ if(DORA_ROOT_DIR) ) FetchContent_MakeAvailable(Corrosion) list(PREPEND CMAKE_MODULE_PATH ${Corrosion_SOURCE_DIR}/cmake) - find_package(Rust 1.70 REQUIRED MODULE) + find_package(Rust 1.72 REQUIRED MODULE) corrosion_import_crate(MANIFEST_PATH "${DORA_ROOT_DIR}/Cargo.toml" CRATES dora-node-api-c diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 0538cafa..e724aeb2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.70" +channel = "1.72" components = ["rustfmt", "clippy"] From 4978f800a647e4b88d4010b630ad3c9c4d187156 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 21:02:32 +0100 Subject: [PATCH 048/104] Fix clippy and rust warnings --- Cargo.lock | 8 ++++---- apis/rust/operator/src/raw.rs | 8 +++++--- apis/rust/operator/types/Cargo.toml | 2 +- binaries/daemon/src/spawn.rs | 3 +-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 518a1d85..0577045c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4881,9 +4881,9 @@ checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" [[package]] name = "safer-ffi" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c1d19b288ca9898cd421c7b105fb7269918a7f8e9253a991e228981ca421ad" +checksum = "395ace5aff9629c7268ca8255aceb945525b2cb644015f3caec5131a6a537c11" dependencies = [ "inventory 0.1.11", "inventory 0.3.12", @@ -4899,9 +4899,9 @@ dependencies = [ [[package]] name = "safer_ffi-proc_macros" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d7a04caa3ca2224f5ea4ddd850e2629c3b36b2b83621f87a8303bf41020110" +checksum = "9255504d5467bae9e07d58b8de446ba6739b29bf72e1fa35b2387e30d29dcbfe" dependencies = [ "macro_rules_attribute", "prettyplease", diff --git a/apis/rust/operator/src/raw.rs b/apis/rust/operator/src/raw.rs index 9a05a5e3..46aa9638 100644 --- a/apis/rust/operator/src/raw.rs +++ b/apis/rust/operator/src/raw.rs @@ -24,7 +24,7 @@ pub unsafe fn dora_init_operator() -> DoraInitResult { pub unsafe fn dora_drop_operator(operator_context: *mut c_void) -> DoraResult { let raw: *mut O = operator_context.cast(); - unsafe { Box::from_raw(raw) }; + let _ = unsafe { Box::from_raw(raw) }; DoraResult { error: None } } @@ -40,10 +40,12 @@ pub unsafe fn dora_on_event( let event_variant = if let Some(input) = &mut event.input { let Some(data_array) = input.data_array.take() else { return OnEventResult { - result: DoraResult { error: Some("data already taken".to_string().into()) }, + result: DoraResult { + error: Some("data already taken".to_string().into()), + }, status: DoraStatus::Continue, }; - }; + }; let data = arrow::ffi::from_ffi(data_array, &input.schema); match data { diff --git a/apis/rust/operator/types/Cargo.toml b/apis/rust/operator/types/Cargo.toml index 499a02ed..bfc281c4 100644 --- a/apis/rust/operator/types/Cargo.toml +++ b/apis/rust/operator/types/Cargo.toml @@ -13,5 +13,5 @@ arrow = { workspace = true, features = ["ffi"] } dora-arrow-convert = { workspace = true } [dependencies.safer-ffi] -version = "0.1.3" +version = "0.1.4" features = ["headers", "inventory-0-3-1"] diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 1a7e54c3..284899ce 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -93,8 +93,7 @@ pub async fn spawn_node( } _ => { tracing::info!("spawning: {}", resolved_path.display()); - let cmd = tokio::process::Command::new(&resolved_path); - cmd + tokio::process::Command::new(&resolved_path) } }; From dec4b0f87d4d83a2f4b4015b9a52ab1a0a460a17 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 21:05:27 +0100 Subject: [PATCH 049/104] Bump to 0.3.1-rc6 --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- Changelog.md | 2 +- apis/python/node/dora/__init__.py | 2 +- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0577045c..7e71ea44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -1013,7 +1013,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "flume", "zenoh", @@ -1021,7 +1021,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1-rc5" +version = "0.3.1-rc6" [[package]] name = "concurrent-queue" @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "arrow", "eyre", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1460,7 +1460,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "aligned-vec", "dora-message", @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "aligned-vec", "arrow-schema", @@ -1508,7 +1508,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "eyre", "reqwest", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "arrow-data", "arrow-schema", @@ -1550,7 +1550,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1561,7 +1561,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "aligned-vec", "arrow", @@ -1590,7 +1590,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "arrow-array", "dora-node-api", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "cxx", "cxx-build", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "arrow", "dora-node-api", @@ -1628,7 +1628,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1637,14 +1637,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "cxx", "cxx-build", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "proc-macro2", "quote", @@ -1662,7 +1662,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "aligned-vec", "arrow", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "arrow", "dora-arrow-convert", @@ -1685,7 +1685,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "chrono", "dora-node-api", @@ -1758,7 +1758,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "aligned-vec", "arrow", @@ -1790,7 +1790,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -3280,14 +3280,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -4688,7 +4688,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -4699,14 +4699,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "eyre", @@ -4714,7 +4714,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5118,7 +5118,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1-rc5" +version = "0.3.1-rc6" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index e1557eb3..d4961d30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1-rc5" +version = "0.3.1-rc6" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1-rc5", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1-rc5", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1-rc5", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1-rc5", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1-rc5", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1-rc5", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1-rc5", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1-rc5", path = "apis/c/node" } -dora-core = { version = "0.3.1-rc5", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1-rc5", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1-rc5", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1-rc5", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1-rc5", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1-rc5", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1-rc5", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1-rc5", path = "libraries/message" } -dora-runtime = { version = "0.3.1-rc5", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1-rc5", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1-rc5", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1-rc6", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1-rc6", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1-rc6", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1-rc6", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1-rc6", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1-rc6", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1-rc6", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1-rc6", path = "apis/c/node" } +dora-core = { version = "0.3.1-rc6", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1-rc6", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1-rc6", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1-rc6", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1-rc6", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1-rc6", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1-rc6", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1-rc6", path = "libraries/message" } +dora-runtime = { version = "0.3.1-rc6", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1-rc6", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1-rc6", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/Changelog.md b/Changelog.md index d0e758ab..ee0fc100 100644 --- a/Changelog.md +++ b/Changelog.md @@ -24,7 +24,7 @@ - @XxChang made their first contribution in https://github.com/dora-rs/dora/pull/381 -**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.0...v0.3.1-rc5 +**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.3.0...v0.3.1 ## v0.3.0 (2023-11-01) diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 27aea294..d586aa46 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.1-rc5" +__version__ = "0.3.1-rc6" class DoraStatus(Enum): From fd7c0939df5877c5d1ae6ead493e8c8c56922438 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 21:08:23 +0100 Subject: [PATCH 050/104] Run `cargo fmt --all` --- apis/c++/node/src/lib.rs | 7 ++++++- binaries/coordinator/src/lib.rs | 7 +++++-- binaries/daemon/src/node_communication/mod.rs | 6 +++++- binaries/runtime/src/lib.rs | 4 +++- binaries/runtime/src/operator/python.rs | 4 +++- binaries/runtime/src/operator/shared_lib.rs | 2 +- libraries/extensions/ros2-bridge/python/src/lib.rs | 2 +- 7 files changed, 24 insertions(+), 8 deletions(-) diff --git a/apis/c++/node/src/lib.rs b/apis/c++/node/src/lib.rs index 77fd3197..61220ecf 100644 --- a/apis/c++/node/src/lib.rs +++ b/apis/c++/node/src/lib.rs @@ -82,7 +82,12 @@ fn event_type(event: &DoraEvent) -> ffi::DoraEventType { } fn event_as_input(event: Box) -> eyre::Result { - let Some(Event::Input { id, metadata: _, data }) = event.0 else { + let Some(Event::Input { + id, + metadata: _, + data, + }) = event.0 + else { bail!("not an input event"); }; let data: Option<&BinaryArray> = data.as_binary_opt(); diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index 72492c70..febb7b0f 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -251,8 +251,11 @@ async fn start_inner( // notify all machines that run parts of the dataflow for machine_id in &dataflow.machines { - let Some(connection) = daemon_connections.get_mut(machine_id) else { - tracing::warn!("no daemon connection found for machine `{machine_id}`"); + let Some(connection) = daemon_connections.get_mut(machine_id) + else { + tracing::warn!( + "no daemon connection found for machine `{machine_id}`" + ); continue; }; tcp_send(&mut connection.stream, &message) diff --git a/binaries/daemon/src/node_communication/mod.rs b/binaries/daemon/src/node_communication/mod.rs index 5af17d18..8fd200de 100644 --- a/binaries/daemon/src/node_communication/mod.rs +++ b/binaries/daemon/src/node_communication/mod.rs @@ -299,7 +299,11 @@ impl Listener { // iterate over queued events, newest first for event in self.queue.iter_mut().rev() { - let Some(Timestamped { inner: NodeEvent::Input { id, data, .. }, ..}) = event.as_mut() else { + let Some(Timestamped { + inner: NodeEvent::Input { id, data, .. }, + .. + }) = event.as_mut() + else { continue; }; match queue_size_remaining.get_mut(id) { diff --git a/binaries/runtime/src/lib.rs b/binaries/runtime/src/lib.rs index e903581b..9d903889 100644 --- a/binaries/runtime/src/lib.rs +++ b/binaries/runtime/src/lib.rs @@ -190,7 +190,9 @@ async fn run( } let Some(config) = operators.get(&operator_id) else { - tracing::warn!("received Finished event for unknown operator `{operator_id}`"); + tracing::warn!( + "received Finished event for unknown operator `{operator_id}`" + ); continue; }; let outputs = config diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index cc555aa5..5b2b12df 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -123,7 +123,9 @@ pub fn run( let mut reload = false; let reason = loop { #[allow(unused_mut)] - let Ok(mut event) = incoming_events.recv() else { break StopReason::InputsClosed }; + let Ok(mut event) = incoming_events.recv() else { + break StopReason::InputsClosed; + }; if let Event::Reload { .. } = event { reload = true; diff --git a/binaries/runtime/src/operator/shared_lib.rs b/binaries/runtime/src/operator/shared_lib.rs index ca758c4f..6a91102c 100644 --- a/binaries/runtime/src/operator/shared_lib.rs +++ b/binaries/runtime/src/operator/shared_lib.rs @@ -168,7 +168,7 @@ impl<'lib> SharedLibraryOperator<'lib> { let reason = loop { #[allow(unused_mut)] let Ok(mut event) = self.incoming_events.recv() else { - break StopReason::InputsClosed + break StopReason::InputsClosed; }; let span = span!(tracing::Level::TRACE, "on_event", input_id = field::Empty); diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 8b3abb11..37ac68a6 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -238,7 +238,7 @@ impl Ros2Subscription { .take_seed(self.deserializer.clone()) .context("failed to take next message from subscription")?; let Some((value, _info)) = message else { - return Ok(None) + return Ok(None); }; let message = value.to_pyarrow(py)?; From afae2bdece26a8773a0d28068cb90b8512209de8 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Jan 2024 21:59:57 +0100 Subject: [PATCH 051/104] Bump to version 0.3.1 --- Cargo.lock | 66 ++++++++++++++--------------- Cargo.toml | 40 ++++++++--------- README.md | 12 +++--- apis/python/node/dora/__init__.py | 2 +- apis/rust/operator/types/src/lib.rs | 1 + 5 files changed, 61 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e71ea44..50aa206d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -1013,7 +1013,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "flume", "zenoh", @@ -1021,7 +1021,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1-rc6" +version = "0.3.1" [[package]] name = "concurrent-queue" @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "arrow", "eyre", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "clap 4.4.6", "ctrlc", @@ -1460,7 +1460,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "aligned-vec", "dora-message", @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "aligned-vec", "arrow-schema", @@ -1508,7 +1508,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "eyre", "reqwest", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "arrow-data", "arrow-schema", @@ -1550,7 +1550,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "futures", "opentelemetry 0.21.0", @@ -1561,7 +1561,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "aligned-vec", "arrow", @@ -1590,7 +1590,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "arrow-array", "dora-node-api", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "cxx", "cxx-build", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "arrow", "dora-node-api", @@ -1628,7 +1628,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1637,14 +1637,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "cxx", "cxx-build", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "proc-macro2", "quote", @@ -1662,7 +1662,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "aligned-vec", "arrow", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "arrow", "dora-arrow-convert", @@ -1685,7 +1685,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "chrono", "dora-node-api", @@ -1758,7 +1758,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "aligned-vec", "arrow", @@ -1790,7 +1790,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -3280,14 +3280,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -4688,7 +4688,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -4699,14 +4699,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "eyre", @@ -4714,7 +4714,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5118,7 +5118,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1-rc6" +version = "0.3.1" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index d4961d30..4d13bf32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1-rc6" +version = "0.3.1" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1-rc6", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1-rc6", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1-rc6", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1-rc6", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1-rc6", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1-rc6", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1-rc6", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1-rc6", path = "apis/c/node" } -dora-core = { version = "0.3.1-rc6", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1-rc6", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1-rc6", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1-rc6", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1-rc6", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1-rc6", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1-rc6", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1-rc6", path = "libraries/message" } -dora-runtime = { version = "0.3.1-rc6", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1-rc6", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1-rc6", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.1", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.1", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.1", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.1", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.1", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.1", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.1", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.1", path = "apis/c/node" } +dora-core = { version = "0.3.1", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.1", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.1", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.1", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.1", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.1", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.1", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.1", path = "libraries/message" } +dora-runtime = { version = "0.3.1", path = "binaries/runtime" } +dora-daemon = { version = "0.3.1", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.1", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/README.md b/README.md index c1cb9b30..a51d2fcc 100644 --- a/README.md +++ b/README.md @@ -71,17 +71,17 @@ For more installation guideline, check out our installation guide here: https:// 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1-rc5/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index d586aa46..c5351f50 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.1-rc6" +__version__ = "0.3.1" class DoraStatus(Enum): diff --git a/apis/rust/operator/types/src/lib.rs b/apis/rust/operator/types/src/lib.rs index af44c5be..e92d4538 100644 --- a/apis/rust/operator/types/src/lib.rs +++ b/apis/rust/operator/types/src/lib.rs @@ -1,4 +1,5 @@ #![deny(elided_lifetimes_in_paths)] // required for safer-ffi +#![allow(improper_ctypes_definitions)] pub use arrow; use dora_arrow_convert::{ArrowData, IntoArrow}; From 48534c9af244486970f0ecfc51dcd74defcdd481 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 19:32:54 +0100 Subject: [PATCH 052/104] Removing runtime from the daemon --- binaries/daemon/src/lib.rs | 4 ---- binaries/daemon/src/main.rs | 8 -------- 2 files changed, 12 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index bb806859..3fb9781c 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -1062,10 +1062,6 @@ impl Daemon { } } -pub fn run_dora_runtime() -> eyre::Result<()> { - dora_runtime::main() -} - async fn send_output_to_local_receivers( node_id: NodeId, output_id: DataId, diff --git a/binaries/daemon/src/main.rs b/binaries/daemon/src/main.rs index 1c40c53e..c38bcbf8 100644 --- a/binaries/daemon/src/main.rs +++ b/binaries/daemon/src/main.rs @@ -25,9 +25,6 @@ pub struct Args { #[clap(long)] pub run_dataflow: Option, - - #[clap(long)] - pub run_dora_runtime: bool, } #[tokio::main] @@ -42,13 +39,8 @@ async fn run() -> eyre::Result<()> { run_dataflow, machine_id, coordinator_addr, - run_dora_runtime, } = clap::Parser::parse(); - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - #[cfg(feature = "tracing")] set_up_tracing("dora-daemon").wrap_err("failed to set up tracing subscriber")?; From 6dfd4b17b3a9a6886581b8da06d6c12ecfd053f3 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 19:33:38 +0100 Subject: [PATCH 053/104] adding daemon, coordinator and runtime command in the cli --- Cargo.lock | 7 +- binaries/cli/Cargo.toml | 6 ++ binaries/cli/src/main.rs | 134 ++++++++++++++++++++++++++++++-- binaries/coordinator/src/lib.rs | 6 +- binaries/daemon/Cargo.toml | 1 - 5 files changed, 144 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50aa206d..4efc9561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,17 +1417,23 @@ dependencies = [ "clap 4.4.6", "communication-layer-request-reply", "ctrlc", + "dora-coordinator", "dora-core", + "dora-daemon", "dora-node-api-c", "dora-operator-api-c", + "dora-runtime", "dora-tracing", "eyre", + "futures", "inquire", "notify", "serde", "serde_json", "serde_yaml 0.9.25", "termcolor", + "tokio", + "tokio-stream", "tracing", "uuid", "webbrowser", @@ -1488,7 +1494,6 @@ dependencies = [ "ctrlc", "dora-core", "dora-download", - "dora-runtime", "dora-tracing", "eyre", "flume", diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index aa3b23f2..0990b9e3 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -35,3 +35,9 @@ ctrlc = "3.2.5" tracing = "0.1.36" dora-tracing = { workspace = true, optional = true } bat = "0.23.0" +dora-daemon = { workspace = true } +dora-coordinator = { workspace = true } +dora-runtime = { workspace = true } +tokio = { version = "1.20.1", features = ["full"] } +tokio-stream = { version = "0.1.8", features = ["io-util", "net"] } +futures = "0.3.21" diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index 96e4435d..f072237e 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -1,15 +1,25 @@ -use std::path::PathBuf; +use std::{net::Ipv4Addr, path::PathBuf}; use attach::attach_dataflow; use clap::Parser; use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection}; use dora_core::{ + daemon_messages::Timestamped, descriptor::Descriptor, - topics::{control_socket_addr, ControlRequest, ControlRequestReply, DataflowId}, + message::uhlc::HLC, + topics::{ + control_socket_addr, ControlRequest, ControlRequestReply, DataflowId, + DORA_COORDINATOR_PORT_DEFAULT, + }, }; +use dora_daemon::Daemon; #[cfg(feature = "tracing")] use dora_tracing::set_up_tracing; use eyre::{bail, Context}; +use futures::{Stream, StreamExt}; +use std::net::SocketAddr; +use tokio::{runtime::Builder, sync::mpsc}; +use tokio_stream::wrappers::ReceiverStream; use uuid::Uuid; mod attach; @@ -92,6 +102,24 @@ enum Command { // Stats, // Get, // Upgrade, + /// Run daemon + Daemon { + #[clap(long)] + machine_id: Option, + #[clap(long)] + coordinator_addr: Option, + + #[clap(long)] + run_dataflow: Option, + }, + /// Run runtime + Runtime, + /// Run coordinator + Coordinator { port: Option }, +} + +enum Event { + CtrlC, } #[derive(Debug, clap::Args)] @@ -127,10 +155,26 @@ fn main() { } fn run() -> eyre::Result<()> { - #[cfg(feature = "tracing")] - set_up_tracing("dora-cli").context("failed to set up tracing subscriber")?; let args = Args::parse(); + #[cfg(feature = "tracing")] + match args.command { + Command::Daemon { .. } => { + set_up_tracing("dora-daemon").context("failed to set up tracing subscriber")?; + } + Command::Runtime => { + set_up_tracing("dora-runtime").context("failed to set up tracing subscriber")?; + } + Command::Coordinator { .. } => { + set_up_tracing("dora-coordinator").context("failed to set up tracing subscriber")?; + } + _ => { + set_up_tracing("dora-cli").context("failed to set up tracing subscriber")?; + } + }; + + let ctrlc_events = set_up_ctrlc_handler()?; + match args.command { Command::Check { dataflow } => match dataflow { Some(dataflow) => { @@ -227,7 +271,65 @@ fn run() -> eyre::Result<()> { } } Command::Destroy { config } => up::destroy(config.as_deref())?, - } + Command::Coordinator { port } => { + let rt = Builder::new_multi_thread() + .enable_all() + .build() + .context("tokio runtime failed")?; + rt.block_on(async { + let (_, task) = dora_coordinator::start( + port, + ctrlc_events.map(|event| match event { + Event::CtrlC => dora_coordinator::Event::CtrlC, + }), + ) + .await?; + task.await + }) + .context("failed to run dora-coordinator")? + } + Command::Daemon { + coordinator_addr, + machine_id, + run_dataflow, + } => { + let rt = Builder::new_multi_thread() + .enable_io() + .build() + .context("tokio runtime failed")?; + rt.block_on(async { + match run_dataflow { + Some(dataflow_path) => { + tracing::info!("Starting dataflow `{}`", dataflow_path.display()); + + Daemon::run_dataflow(&dataflow_path).await + } + None => { + Daemon::run( + coordinator_addr.unwrap_or_else(|| { + tracing::info!("Starting in local mode"); + let localhost = Ipv4Addr::new(127, 0, 0, 1); + (localhost, DORA_COORDINATOR_PORT_DEFAULT).into() + }), + machine_id.unwrap_or_default(), + ctrlc_events.map(|event| { + let clock = HLC::default(); + match event { + Event::CtrlC => Timestamped { + inner: dora_daemon::Event::CtrlC, + timestamp: clock.new_timestamp(), + }, + } + }), + ) + .await + } + } + }) + .context("failed to run dora-daemon")? + } + Command::Runtime => dora_runtime::main().context("Failed to run dora-runtime")?, + }; Ok(()) } @@ -349,3 +451,25 @@ fn query_running_dataflows( fn connect_to_coordinator() -> std::io::Result> { TcpLayer::new().connect(control_socket_addr()) } + +fn set_up_ctrlc_handler() -> Result, eyre::ErrReport> { + let (ctrlc_tx, ctrlc_rx) = mpsc::channel(1); + + let mut ctrlc_sent = false; + ctrlc::set_handler(move || { + if ctrlc_sent { + tracing::warn!("received second ctrlc signal -> aborting immediately"); + std::process::abort(); + } else { + tracing::info!("received ctrlc signal"); + if ctrlc_tx.blocking_send(Event::CtrlC).is_err() { + tracing::error!("failed to report ctrl-c event to dora-coordinator"); + } + + ctrlc_sent = true; + } + }) + .wrap_err("failed to set ctrl-c handler")?; + + Ok(ReceiverStream::new(ctrlc_rx)) +} diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index febb7b0f..056dade8 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -48,7 +48,7 @@ pub struct Args { pub async fn run(args: Args) -> eyre::Result<()> { let ctrlc_events = set_up_ctrlc_handler()?; - let (_, task) = start(args, ctrlc_events).await?; + let (_, task) = start(None, ctrlc_events).await?; task.await?; @@ -56,10 +56,10 @@ pub async fn run(args: Args) -> eyre::Result<()> { } pub async fn start( - args: Args, + port: Option, external_events: impl Stream + Unpin, ) -> Result<(u16, impl Future>), eyre::ErrReport> { - let port = args.port.unwrap_or(DORA_COORDINATOR_PORT_DEFAULT); + let port = port.unwrap_or(DORA_COORDINATOR_PORT_DEFAULT); let listener = listener::create_listener(port).await?; let port = listener .local_addr() diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index a5ba2491..43255a1f 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -25,7 +25,6 @@ futures-concurrency = "7.1.0" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.86" dora-core = { workspace = true } -dora-runtime = { workspace = true } flume = "0.10.14" dora-download = { workspace = true } dora-tracing = { workspace = true, optional = true } From f85011837dc67b46bc603c7bb73a5183c7e6f24c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 19:38:38 +0100 Subject: [PATCH 054/104] Remove unused runtime args from examples --- examples/benchmark/run.rs | 12 ------------ examples/c++-dataflow/run.rs | 11 ----------- examples/c-dataflow/run.rs | 12 ------------ examples/cmake-dataflow/run.rs | 11 ----------- examples/multiple-daemons/run.rs | 18 ++---------------- examples/rust-dataflow-url/run.rs | 11 ----------- examples/rust-dataflow/run.rs | 12 ------------ examples/rust-ros2-dataflow/run.rs | 12 ------------ 8 files changed, 2 insertions(+), 97 deletions(-) diff --git a/examples/benchmark/run.rs b/examples/benchmark/run.rs index 54c8ac76..16df626e 100644 --- a/examples/benchmark/run.rs +++ b/examples/benchmark/run.rs @@ -2,20 +2,8 @@ use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("benchmark-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 6915c73f..5f17138f 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -5,19 +5,8 @@ use std::{ path::Path, }; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } set_up_tracing("c++-dataflow-runner").wrap_err("failed to set up tracing")?; if cfg!(windows) { diff --git a/examples/c-dataflow/run.rs b/examples/c-dataflow/run.rs index 4268c25c..4ebcfdb5 100644 --- a/examples/c-dataflow/run.rs +++ b/examples/c-dataflow/run.rs @@ -5,20 +5,8 @@ use std::{ path::Path, }; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("c-dataflow-runner").wrap_err("failed to set up tracing")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/examples/cmake-dataflow/run.rs b/examples/cmake-dataflow/run.rs index 6ede3e72..27cbc68c 100644 --- a/examples/cmake-dataflow/run.rs +++ b/examples/cmake-dataflow/run.rs @@ -2,19 +2,8 @@ use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } set_up_tracing("cmake-dataflow-runner").wrap_err("failed to set up tracing")?; if cfg!(windows) { diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index d3499382..c3b7dddd 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -22,19 +22,8 @@ use tokio::{ use tokio_stream::wrappers::ReceiverStream; use uuid::Uuid; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("multiple-daemon-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); @@ -45,11 +34,8 @@ async fn main() -> eyre::Result<()> { build_dataflow(dataflow).await?; let (coordinator_events_tx, coordinator_events_rx) = mpsc::channel(1); - let (coordinator_port, coordinator) = dora_coordinator::start( - dora_coordinator::Args { port: Some(0) }, - ReceiverStream::new(coordinator_events_rx), - ) - .await?; + let (coordinator_port, coordinator) = + dora_coordinator::start(Some(0), ReceiverStream::new(coordinator_events_rx)).await?; let coordinator_addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), coordinator_port); let daemon_a = dora_daemon::Daemon::run(coordinator_addr, "A".into(), stream::empty()); let daemon_b = dora_daemon::Daemon::run(coordinator_addr, "B".into(), stream::empty()); diff --git a/examples/rust-dataflow-url/run.rs b/examples/rust-dataflow-url/run.rs index 7e16d3b9..c7c3d847 100644 --- a/examples/rust-dataflow-url/run.rs +++ b/examples/rust-dataflow-url/run.rs @@ -2,19 +2,8 @@ use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("rust-dataflow-url-runner").wrap_err("failed to set up tracing")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/examples/rust-dataflow/run.rs b/examples/rust-dataflow/run.rs index 2fbc50d9..82095d03 100644 --- a/examples/rust-dataflow/run.rs +++ b/examples/rust-dataflow/run.rs @@ -2,20 +2,8 @@ use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("rust-dataflow-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/examples/rust-ros2-dataflow/run.rs b/examples/rust-ros2-dataflow/run.rs index cb999d8c..5818e03e 100644 --- a/examples/rust-ros2-dataflow/run.rs +++ b/examples/rust-ros2-dataflow/run.rs @@ -2,20 +2,8 @@ use dora_tracing::set_up_tracing; use eyre::{bail, Context}; use std::path::Path; -#[derive(Debug, Clone, clap::Parser)] -pub struct Args { - #[clap(long)] - pub run_dora_runtime: bool, -} - #[tokio::main] async fn main() -> eyre::Result<()> { - let Args { run_dora_runtime } = clap::Parser::parse(); - - if run_dora_runtime { - return tokio::task::block_in_place(dora_daemon::run_dora_runtime); - } - set_up_tracing("rust-ros2-dataflow-runner").wrap_err("failed to set up tracing subscriber")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); From 54b0882f6f4183272ccfc30692e810c3d4ffe0d6 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 20:49:06 +0100 Subject: [PATCH 055/104] link up command with the newly added binaries within the cli --- binaries/cli/src/main.rs | 16 ++-------------- binaries/cli/src/up.rs | 30 +++++++++++++----------------- binaries/daemon/src/spawn.rs | 2 +- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index f072237e..fb8131c1 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -66,10 +66,6 @@ enum Command { Up { #[clap(long)] config: Option, - #[clap(long)] - coordinator_path: Option, - #[clap(long)] - daemon_path: Option, }, /// Destroy running coordinator and daemon. If some dataflows are still running, they will be stopped first. Destroy { @@ -203,15 +199,7 @@ fn run() -> eyre::Result<()> { args, internal_create_with_path_dependencies, } => template::create(args, internal_create_with_path_dependencies)?, - Command::Up { - config, - coordinator_path, - daemon_path, - } => up::up( - config.as_deref(), - coordinator_path.as_deref(), - daemon_path.as_deref(), - )?, + Command::Up { config } => up::up(config.as_deref())?, Command::Logs { dataflow, node } => { let uuid = Uuid::parse_str(&dataflow).ok(); let name = if uuid.is_some() { None } else { Some(dataflow) }; @@ -294,7 +282,7 @@ fn run() -> eyre::Result<()> { run_dataflow, } => { let rt = Builder::new_multi_thread() - .enable_io() + .enable_all() .build() .context("tokio runtime failed")?; rt.block_on(async { diff --git a/binaries/cli/src/up.rs b/binaries/cli/src/up.rs index 7d551a74..b591499f 100644 --- a/binaries/cli/src/up.rs +++ b/binaries/cli/src/up.rs @@ -6,17 +6,13 @@ use std::{fs, path::Path, process::Command, time::Duration}; #[derive(Debug, Default, serde::Serialize, serde::Deserialize)] struct UpConfig {} -pub(crate) fn up( - config_path: Option<&Path>, - coordinator: Option<&Path>, - daemon: Option<&Path>, -) -> eyre::Result<()> { +pub(crate) fn up(config_path: Option<&Path>) -> eyre::Result<()> { let UpConfig {} = parse_dora_config(config_path)?; let mut session = match connect_to_coordinator() { Ok(session) => session, Err(_) => { - start_coordinator(coordinator).wrap_err("failed to start dora-coordinator")?; + start_coordinator().wrap_err("failed to start dora-coordinator")?; loop { match connect_to_coordinator() { @@ -31,7 +27,7 @@ pub(crate) fn up( }; if !daemon_running(&mut *session)? { - start_daemon(daemon).wrap_err("failed to start dora-daemon")?; + start_daemon().wrap_err("failed to start dora-daemon")?; } Ok(()) @@ -70,24 +66,24 @@ fn parse_dora_config(config_path: Option<&Path>) -> Result) -> eyre::Result<()> { - let coordinator = coordinator.unwrap_or_else(|| Path::new("dora-coordinator")); - - let mut cmd = Command::new(coordinator); +fn start_coordinator() -> eyre::Result<()> { + let mut cmd = + Command::new(std::env::current_exe().wrap_err("failed to get current executable path")?); + cmd.arg("coordinator"); cmd.spawn() - .wrap_err_with(|| format!("failed to run {}", coordinator.display()))?; + .wrap_err_with(|| format!("failed to run `dora coordinator`"))?; println!("started dora coordinator"); Ok(()) } -fn start_daemon(daemon: Option<&Path>) -> eyre::Result<()> { - let daemon = daemon.unwrap_or_else(|| Path::new("dora-daemon")); - - let mut cmd = Command::new(daemon); +fn start_daemon() -> eyre::Result<()> { + let mut cmd = + Command::new(std::env::current_exe().wrap_err("failed to get current executable path")?); + cmd.arg("daemon"); cmd.spawn() - .wrap_err_with(|| format!("failed to run {}", daemon.display()))?; + .wrap_err_with(|| format!("failed to run `dora daemon`"))?; println!("started dora daemon"); diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 284899ce..ccefdbed 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -162,7 +162,7 @@ pub async fn spawn_node( let mut cmd = tokio::process::Command::new( std::env::current_exe().wrap_err("failed to get current executable path")?, ); - cmd.arg("--run-dora-runtime"); + cmd.arg("runtime"); cmd } else { eyre::bail!("Runtime can not mix Python Operator with other type of operator."); From 4ec81cc74322350294a7b234ed592a8b55a9dd48 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 20:54:00 +0100 Subject: [PATCH 056/104] Removing old main files --- Cargo.lock | 7 --- binaries/coordinator/Cargo.toml | 3 -- binaries/coordinator/src/lib.rs | 39 -------------- binaries/coordinator/src/main.rs | 12 ----- binaries/daemon/Cargo.toml | 3 -- binaries/daemon/src/main.rs | 90 -------------------------------- binaries/runtime/Cargo.toml | 1 - 7 files changed, 155 deletions(-) delete mode 100644 binaries/coordinator/src/main.rs delete mode 100644 binaries/daemon/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 4efc9561..058b53f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,8 +1443,6 @@ dependencies = [ name = "dora-coordinator" version = "0.3.1" dependencies = [ - "clap 4.4.6", - "ctrlc", "dora-core", "dora-tracing", "eyre", @@ -1460,7 +1458,6 @@ dependencies = [ "tokio-stream", "tracing", "uuid", - "which", "zenoh", ] @@ -1490,8 +1487,6 @@ dependencies = [ "arrow-schema", "async-trait", "bincode", - "clap 4.4.6", - "ctrlc", "dora-core", "dora-download", "dora-tracing", @@ -1508,7 +1503,6 @@ dependencies = [ "tracing", "tracing-opentelemetry", "uuid", - "which", ] [[package]] @@ -1768,7 +1762,6 @@ dependencies = [ "aligned-vec", "arrow", "arrow-schema", - "clap 4.4.6", "dora-core", "dora-download", "dora-metrics", diff --git a/binaries/coordinator/Cargo.toml b/binaries/coordinator/Cargo.toml index 0692ce4f..6da51659 100644 --- a/binaries/coordinator/Cargo.toml +++ b/binaries/coordinator/Cargo.toml @@ -27,8 +27,5 @@ dora-tracing = { workspace = true, optional = true } futures-concurrency = "7.1.0" zenoh = "0.7.0-rc" serde_json = "1.0.86" -which = "5.0.0" thiserror = "1.0.37" -ctrlc = "3.2.5" -clap = { version = "4.0.3", features = ["derive"] } names = "0.14.0" diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index 056dade8..f167bba2 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -38,23 +38,6 @@ mod listener; mod run; mod tcp_utils; -#[derive(Debug, Clone, clap::Parser)] -#[clap(about = "Dora coordinator")] -pub struct Args { - #[clap(long)] - pub port: Option, -} - -pub async fn run(args: Args) -> eyre::Result<()> { - let ctrlc_events = set_up_ctrlc_handler()?; - - let (_, task) = start(None, ctrlc_events).await?; - - task.await?; - - Ok(()) -} - pub async fn start( port: Option, external_events: impl Stream + Unpin, @@ -604,28 +587,6 @@ struct DaemonConnection { last_heartbeat: Instant, } -fn set_up_ctrlc_handler() -> Result, eyre::ErrReport> { - let (ctrlc_tx, ctrlc_rx) = mpsc::channel(1); - - let mut ctrlc_sent = false; - ctrlc::set_handler(move || { - if ctrlc_sent { - tracing::warn!("received second ctrlc signal -> aborting immediately"); - std::process::abort(); - } else { - tracing::info!("received ctrlc signal"); - if ctrlc_tx.blocking_send(Event::CtrlC).is_err() { - tracing::error!("failed to report ctrl-c event to dora-coordinator"); - } - - ctrlc_sent = true; - } - }) - .wrap_err("failed to set ctrl-c handler")?; - - Ok(ReceiverStream::new(ctrlc_rx)) -} - async fn handle_destroy( running_dataflows: &HashMap, daemon_connections: &mut HashMap, diff --git a/binaries/coordinator/src/main.rs b/binaries/coordinator/src/main.rs deleted file mode 100644 index 705b4942..00000000 --- a/binaries/coordinator/src/main.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[cfg(feature = "tracing")] -use dora_tracing::set_up_tracing; -#[cfg(feature = "tracing")] -use eyre::Context; - -#[tokio::main] -async fn main() -> eyre::Result<()> { - #[cfg(feature = "tracing")] - set_up_tracing("dora-coordinator").context("failed to set up tracing subscriber")?; - - dora_coordinator::run(clap::Parser::parse()).await -} diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index 43255a1f..668f5562 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -31,11 +31,8 @@ dora-tracing = { workspace = true, optional = true } serde_yaml = "0.8.23" uuid = { version = "1.1.2", features = ["v4"] } futures = "0.3.25" -clap = { version = "4.0.3", features = ["derive"] } shared-memory-server = { workspace = true } -ctrlc = "3.2.5" bincode = "1.3.3" async-trait = "0.1.64" arrow-schema = { workspace = true } aligned-vec = "0.5.0" -which = "5.0.0" diff --git a/binaries/daemon/src/main.rs b/binaries/daemon/src/main.rs deleted file mode 100644 index c38bcbf8..00000000 --- a/binaries/daemon/src/main.rs +++ /dev/null @@ -1,90 +0,0 @@ -use dora_core::{ - daemon_messages::Timestamped, message::uhlc::HLC, topics::DORA_COORDINATOR_PORT_DEFAULT, -}; -use dora_daemon::{Daemon, Event}; -#[cfg(feature = "tracing")] -use dora_tracing::set_up_tracing; -#[cfg(feature = "tracing")] -use eyre::Context; -use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; - -use std::{ - net::{Ipv4Addr, SocketAddr}, - path::PathBuf, -}; - -#[derive(Debug, Clone, clap::Parser)] -#[clap(about = "Dora daemon")] -pub struct Args { - #[clap(long)] - pub machine_id: Option, - - #[clap(long)] - pub coordinator_addr: Option, - - #[clap(long)] - pub run_dataflow: Option, -} - -#[tokio::main] -async fn main() -> eyre::Result<()> { - // the tokio::main proc macro confuses some tools such as rust-analyzer, so - // directly invoke a "normal" async function - run().await -} - -async fn run() -> eyre::Result<()> { - let Args { - run_dataflow, - machine_id, - coordinator_addr, - } = clap::Parser::parse(); - - #[cfg(feature = "tracing")] - set_up_tracing("dora-daemon").wrap_err("failed to set up tracing subscriber")?; - - let ctrl_c_events = { - let (ctrl_c_tx, ctrl_c_rx) = mpsc::channel(1); - let mut ctrlc_sent = false; - ctrlc::set_handler(move || { - let clock = HLC::default(); - if ctrlc_sent { - tracing::warn!("received second ctrlc signal -> aborting immediately"); - std::process::abort(); - } else { - tracing::info!("received ctrlc signal"); - let event = Timestamped { - inner: Event::CtrlC, - timestamp: clock.new_timestamp(), - }; - if ctrl_c_tx.blocking_send(event).is_err() { - tracing::error!("failed to report ctrl-c event to dora-daemon"); - } - ctrlc_sent = true; - } - }) - .wrap_err("failed to set ctrl-c handler")?; - ReceiverStream::new(ctrl_c_rx) - }; - - match run_dataflow { - Some(dataflow_path) => { - tracing::info!("Starting dataflow `{}`", dataflow_path.display()); - - Daemon::run_dataflow(&dataflow_path).await - } - None => { - Daemon::run( - coordinator_addr.unwrap_or_else(|| { - tracing::info!("Starting in local mode"); - let localhost = Ipv4Addr::new(127, 0, 0, 1); - (localhost, DORA_COORDINATOR_PORT_DEFAULT).into() - }), - machine_id.unwrap_or_default(), - ctrl_c_events, - ) - .await - } - } -} diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index 9cf662c1..fb15da62 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -30,7 +30,6 @@ tracing = "0.1.36" tracing-subscriber = "0.3.15" dora-download = { workspace = true } flume = "0.10.14" -clap = { version = "4.0.3", features = ["derive"] } tracing-opentelemetry = { version = "0.18.0", optional = true } pythonize = { workspace = true, optional = true } arrow-schema = { workspace = true } From eb7396eba2198dc16abcb6d9c600047338acd227 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 20:56:46 +0100 Subject: [PATCH 057/104] Update ci --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c2706a3..207a5c12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,8 +236,6 @@ jobs: # fail-fast by using bash shell explictly shell: bash run: | - cargo install --path binaries/coordinator --locked - cargo install --path binaries/daemon --locked cargo install --path binaries/cli --locked - name: "Test CLI" timeout-minutes: 30 From 0568656ef0b1ee75ad2156487ac4116c32be571d Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 21:12:08 +0100 Subject: [PATCH 058/104] Remove unused dependencies for faster compile time --- Cargo.lock | 37 ------------------- apis/c/node/Cargo.toml | 1 - apis/rust/node/Cargo.toml | 7 ---- binaries/coordinator/Cargo.toml | 5 --- binaries/daemon/Cargo.toml | 2 - binaries/runtime/Cargo.toml | 2 - libraries/core/Cargo.toml | 1 - .../extensions/ros2-bridge/python/Cargo.toml | 1 - .../extensions/telemetry/metrics/Cargo.toml | 2 - .../extensions/telemetry/tracing/Cargo.toml | 1 - 10 files changed, 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 058b53f9..6cb9a43b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,12 +814,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" -[[package]] -name = "capnp" -version = "0.14.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dca085c2c7d9d65ad749d450b19b551efaa8e3476a439bdca07aca8533097f3" - [[package]] name = "cc" version = "1.0.83" @@ -1449,16 +1443,11 @@ dependencies = [ "futures", "futures-concurrency", "names", - "rand", - "serde", "serde_json", - "serde_yaml 0.8.26", - "thiserror", "tokio", "tokio-stream", "tracing", "uuid", - "zenoh", ] [[package]] @@ -1471,7 +1460,6 @@ dependencies = [ "once_cell", "serde", "serde-with-expand-env", - "serde_bytes", "serde_yaml 0.9.25", "tokio", "tracing", @@ -1484,7 +1472,6 @@ name = "dora-daemon" version = "0.3.1" dependencies = [ "aligned-vec", - "arrow-schema", "async-trait", "bincode", "dora-core", @@ -1494,7 +1481,6 @@ dependencies = [ "flume", "futures", "futures-concurrency", - "serde", "serde_json", "serde_yaml 0.8.26", "shared-memory-server", @@ -1551,11 +1537,9 @@ dependencies = [ name = "dora-metrics" version = "0.3.1" dependencies = [ - "futures", "opentelemetry 0.21.0", "opentelemetry-otlp", "opentelemetry_sdk 0.21.1", - "tokio", ] [[package]] @@ -1564,9 +1548,7 @@ version = "0.3.1" dependencies = [ "aligned-vec", "arrow", - "arrow-schema", "bincode", - "capnp", "dora-arrow-convert", "dora-core", "dora-tracing", @@ -1575,16 +1557,11 @@ dependencies = [ "futures", "futures-concurrency", "futures-timer", - "once_cell", - "serde", - "serde_json", "serde_yaml 0.8.26", "shared-memory-server", "shared_memory_extended", - "thiserror", "tokio", "tracing", - "uuid", ] [[package]] @@ -1594,7 +1571,6 @@ dependencies = [ "arrow-array", "dora-node-api", "eyre", - "flume", "tracing", ] @@ -1749,7 +1725,6 @@ dependencies = [ "dora-ros2-bridge", "dora-ros2-bridge-msg-gen", "eyre", - "flume", "futures", "pyo3", "serde", @@ -1761,7 +1736,6 @@ version = "0.3.1" dependencies = [ "aligned-vec", "arrow", - "arrow-schema", "dora-core", "dora-download", "dora-metrics", @@ -1783,7 +1757,6 @@ dependencies = [ "tokio-stream", "tracing", "tracing-opentelemetry", - "tracing-subscriber", ] [[package]] @@ -1793,7 +1766,6 @@ dependencies = [ "eyre", "opentelemetry 0.18.0", "opentelemetry-jaeger", - "tokio", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -5005,15 +4977,6 @@ dependencies = [ "shellexpand 2.1.2", ] -[[package]] -name = "serde_bytes" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" version = "1.0.195" diff --git a/apis/c/node/Cargo.toml b/apis/c/node/Cargo.toml index f5346dda..2a601c64 100644 --- a/apis/c/node/Cargo.toml +++ b/apis/c/node/Cargo.toml @@ -19,7 +19,6 @@ tracing = ["dora-node-api/tracing"] [dependencies] eyre = "0.6.8" -flume = "0.10.14" tracing = "0.1.33" arrow-array = { workspace = true } diff --git a/apis/rust/node/Cargo.toml b/apis/rust/node/Cargo.toml index f5fe00bb..015ad9ad 100644 --- a/apis/rust/node/Cargo.toml +++ b/apis/rust/node/Cargo.toml @@ -14,20 +14,13 @@ tracing = ["dep:dora-tracing"] dora-core = { workspace = true } shared-memory-server = { workspace = true } eyre = "0.6.7" -once_cell = "1.13.0" -serde = { version = "1.0.136", features = ["derive"] } serde_yaml = "0.8.23" -serde_json = "1.0.89" -thiserror = "1.0.30" tracing = "0.1.33" flume = "0.10.14" -uuid = { version = "1.1.2", features = ["v4"] } -capnp = "0.14.11" bincode = "1.3.3" shared_memory_extended = "0.13.0" dora-tracing = { workspace = true, optional = true } arrow = { workspace = true } -arrow-schema = { workspace = true } futures = "0.3.28" futures-concurrency = "7.3.0" futures-timer = "3.0.2" diff --git a/binaries/coordinator/Cargo.toml b/binaries/coordinator/Cargo.toml index 6da51659..a1d30cd5 100644 --- a/binaries/coordinator/Cargo.toml +++ b/binaries/coordinator/Cargo.toml @@ -15,17 +15,12 @@ tracing = ["dep:dora-tracing"] [dependencies] eyre = "0.6.7" futures = "0.3.21" -serde = { version = "1.0.136", features = ["derive"] } -serde_yaml = "0.8.23" tokio = { version = "1.24.2", features = ["full"] } tokio-stream = { version = "0.1.8", features = ["io-util", "net"] } uuid = { version = "1.2.1" } -rand = "0.8.5" dora-core = { workspace = true } tracing = "0.1.36" dora-tracing = { workspace = true, optional = true } futures-concurrency = "7.1.0" -zenoh = "0.7.0-rc" serde_json = "1.0.86" -thiserror = "1.0.37" names = "0.14.0" diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index 668f5562..6f6e9154 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -22,7 +22,6 @@ tokio-stream = { version = "0.1.11", features = ["net"] } tracing = "0.1.36" tracing-opentelemetry = { version = "0.18.0", optional = true } futures-concurrency = "7.1.0" -serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.86" dora-core = { workspace = true } flume = "0.10.14" @@ -34,5 +33,4 @@ futures = "0.3.25" shared-memory-server = { workspace = true } bincode = "1.3.3" async-trait = "0.1.64" -arrow-schema = { workspace = true } aligned-vec = "0.5.0" diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index fb15da62..db221f76 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -27,12 +27,10 @@ tokio-stream = "0.1.8" # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html pyo3 = { workspace = true, features = ["eyre", "abi3-py37"], optional = true } tracing = "0.1.36" -tracing-subscriber = "0.3.15" dora-download = { workspace = true } flume = "0.10.14" tracing-opentelemetry = { version = "0.18.0", optional = true } pythonize = { workspace = true, optional = true } -arrow-schema = { workspace = true } arrow = { workspace = true, features = ["ffi"] } aligned-vec = "0.5.0" diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index f4908815..72529189 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -12,7 +12,6 @@ license.workspace = true eyre = "0.6.8" serde = { version = "1.0.136", features = ["derive"] } serde_yaml = "0.9.11" -serde_bytes = "0.11.12" once_cell = "1.13.0" which = "5.0.0" uuid = { version = "1.2.1", features = ["serde"] } diff --git a/libraries/extensions/ros2-bridge/python/Cargo.toml b/libraries/extensions/ros2-bridge/python/Cargo.toml index 8c07908f..5c80cc9b 100644 --- a/libraries/extensions/ros2-bridge/python/Cargo.toml +++ b/libraries/extensions/ros2-bridge/python/Cargo.toml @@ -10,6 +10,5 @@ dora-ros2-bridge-msg-gen = { path = "../msg-gen" } pyo3 = { workspace = true, features = ["eyre", "abi3-py37", "serde"] } eyre = "0.6" serde = "1.0.166" -flume = "0.10.14" arrow = { workspace = true, features = ["pyarrow"] } futures = "0.3.28" diff --git a/libraries/extensions/telemetry/metrics/Cargo.toml b/libraries/extensions/telemetry/metrics/Cargo.toml index 4664da67..53e0ade7 100644 --- a/libraries/extensions/telemetry/metrics/Cargo.toml +++ b/libraries/extensions/telemetry/metrics/Cargo.toml @@ -9,8 +9,6 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -futures = "0.3.21" opentelemetry = { version = "0.21", features = ["metrics"] } opentelemetry-otlp = { version = "0.14.0", features = ["tonic", "metrics"] } -tokio = { version = "1.24.2", features = ["full"] } opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio", "metrics"] } diff --git a/libraries/extensions/telemetry/tracing/Cargo.toml b/libraries/extensions/telemetry/tracing/Cargo.toml index 1b208d58..4808b077 100644 --- a/libraries/extensions/telemetry/tracing/Cargo.toml +++ b/libraries/extensions/telemetry/tracing/Cargo.toml @@ -11,7 +11,6 @@ license.workspace = true [features] [dependencies] -tokio = { version = "1.24.2", features = ["full"] } tracing-subscriber = { version = "0.3.15", features = ["env-filter"] } tracing-opentelemetry = { version = "0.18.0" } eyre = "0.6.8" From 88902cb97b3aad1596099a679f338292263e1ab5 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 21:32:55 +0100 Subject: [PATCH 059/104] Make tracing available for node in the runtime --- binaries/cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index fb8131c1..61f9ba7d 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -159,7 +159,7 @@ fn run() -> eyre::Result<()> { set_up_tracing("dora-daemon").context("failed to set up tracing subscriber")?; } Command::Runtime => { - set_up_tracing("dora-runtime").context("failed to set up tracing subscriber")?; + // Do not set the runtime in the cli. } Command::Coordinator { .. } => { set_up_tracing("dora-coordinator").context("failed to set up tracing subscriber")?; From c72844270a82df5f5475426ba3f6bebda2cc544c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Jan 2024 21:38:46 +0100 Subject: [PATCH 060/104] update CD to ship dora as a single binary ( zipped ) --- .github/workflows/release.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce75ed88..f4fcbf30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,6 @@ jobs: strategy: matrix: platform: [ubuntu-20.04] - python-version: ["3.7"] fail-fast: false runs-on: ${{ matrix.platform }} @@ -72,7 +71,6 @@ jobs: strategy: matrix: platform: [windows-2022] - python-version: ["3.7"] fail-fast: false runs-on: ${{ matrix.platform }} @@ -90,8 +88,6 @@ jobs: shell: powershell run: | New-Item -Path archive -ItemType Directory - Copy-Item target/release/dora-coordinator.exe -Destination archive - Copy-Item target/release/dora-daemon.exe -Destination archive Copy-Item target/release/dora.exe -Destination archive/dora.exe Compress-Archive -Path archive\* -DestinationPath archive.zip @@ -111,7 +107,6 @@ jobs: strategy: matrix: platform: [macos-12, ubuntu-20.04] - python-version: ["3.7"] fail-fast: false runs-on: ${{ matrix.platform }} @@ -128,8 +123,6 @@ jobs: if: runner.os == 'Linux' || runner.os == 'macOS' run: | mkdir archive - cp target/release/dora-coordinator archive - cp target/release/dora-daemon archive cp target/release/dora archive/dora cd archive zip -r ../archive.zip . @@ -151,7 +144,6 @@ jobs: strategy: matrix: platform: [ubuntu-20.04] - python-version: ["3.7"] fail-fast: false runs-on: ${{ matrix.platform }} @@ -172,8 +164,6 @@ jobs: if: runner.os == 'Linux' run: | mkdir archive_aarch64 - cp target/aarch64-unknown-linux-gnu/release/dora-coordinator archive_aarch64 - cp target/aarch64-unknown-linux-gnu/release/dora-daemon archive_aarch64 cp target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora cd archive_aarch64 zip -r ../archive_aarch64.zip . @@ -196,7 +186,6 @@ jobs: strategy: matrix: platform: [macos-12] - python-version: ["3.7"] fail-fast: false runs-on: ${{ matrix.platform }} @@ -219,8 +208,6 @@ jobs: if: runner.os == 'macOS' run: | mkdir archive_aarch64 - cp target/aarch64-apple-darwin/release/dora-coordinator archive_aarch64 - cp target/aarch64-apple-darwin/release/dora-daemon archive_aarch64 cp target/aarch64-apple-darwin/release/dora archive_aarch64/dora cd archive_aarch64 zip -r ../archive_aarch64.zip . From f15f9320dc51580f9e5b5a9555f5e37048a824a0 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 14 Jan 2024 21:36:29 +0100 Subject: [PATCH 061/104] Remove zipping of binary for simpler CD and download --- .github/workflows/release.yml | 51 ++++++----------------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4fcbf30..fa1aa47a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,15 +81,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" - - - name: Create Archive (Windows) - if: runner.os == 'Windows' - shell: powershell - run: | - New-Item -Path archive -ItemType Directory - Copy-Item target/release/dora.exe -Destination archive/dora.exe - Compress-Archive -Path archive\* -DestinationPath archive.zip + run: "cargo build --release -p dora-cli" - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 @@ -97,7 +89,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: archive.zip + asset_path: target/release/dora.exe asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip asset_content_type: application/zip @@ -117,16 +109,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" - - - name: "Create Archive (Unix)" - if: runner.os == 'Linux' || runner.os == 'macOS' - run: | - mkdir archive - cp target/release/dora archive/dora - cd archive - zip -r ../archive.zip . - cd .. + run: "cargo build --release -p dora-cli" - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 @@ -134,7 +117,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: archive.zip + asset_path: target/release/dora asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip asset_content_type: application/zip @@ -158,16 +141,7 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli -p dora-daemon - - - name: "Archive Linux ARM64" - if: runner.os == 'Linux' - run: | - mkdir archive_aarch64 - cp target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora - cd archive_aarch64 - zip -r ../archive_aarch64.zip . - cd .. + args: --release --target aarch64-unknown-linux-gnu -p dora-cli - name: "Upload Linux ARM64 asset" if: runner.os == 'Linux' @@ -176,7 +150,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: archive_aarch64.zip + asset_path: target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip asset_content_type: application/zip @@ -202,16 +176,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli -p dora-daemon - - - name: "Archive macOS ARM64" - if: runner.os == 'macOS' - run: | - mkdir archive_aarch64 - cp target/aarch64-apple-darwin/release/dora archive_aarch64/dora - cd archive_aarch64 - zip -r ../archive_aarch64.zip . - cd .. + args: --release --target aarch64-apple-darwin -p dora-cli - name: "Upload macOS ARM64 asset" if: runner.os == 'macOS' @@ -220,6 +185,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: archive_aarch64.zip + asset_path: target/aarch64-apple-darwin/release/dora archive_aarch64/dora asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip asset_content_type: application/zip From ddf94f709400da60936b28c2c2afa96e23b619ef Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 10:20:31 +0100 Subject: [PATCH 062/104] Adding ctrlc back to their individual repo --- Cargo.lock | 2 ++ binaries/cli/src/main.rs | 51 ++------------------------------- binaries/coordinator/Cargo.toml | 1 + binaries/coordinator/src/lib.rs | 29 +++++++++++++++++-- binaries/daemon/Cargo.toml | 1 + binaries/daemon/src/lib.rs | 40 ++++++++++++++++++++++---- 6 files changed, 67 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cb9a43b..da5514ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1437,6 +1437,7 @@ dependencies = [ name = "dora-coordinator" version = "0.3.1" dependencies = [ + "ctrlc", "dora-core", "dora-tracing", "eyre", @@ -1474,6 +1475,7 @@ dependencies = [ "aligned-vec", "async-trait", "bincode", + "ctrlc", "dora-core", "dora-download", "dora-tracing", diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index 61f9ba7d..bce1c1b0 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -4,9 +4,7 @@ use attach::attach_dataflow; use clap::Parser; use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection}; use dora_core::{ - daemon_messages::Timestamped, descriptor::Descriptor, - message::uhlc::HLC, topics::{ control_socket_addr, ControlRequest, ControlRequestReply, DataflowId, DORA_COORDINATOR_PORT_DEFAULT, @@ -16,10 +14,8 @@ use dora_daemon::Daemon; #[cfg(feature = "tracing")] use dora_tracing::set_up_tracing; use eyre::{bail, Context}; -use futures::{Stream, StreamExt}; use std::net::SocketAddr; -use tokio::{runtime::Builder, sync::mpsc}; -use tokio_stream::wrappers::ReceiverStream; +use tokio::runtime::Builder; use uuid::Uuid; mod attach; @@ -114,10 +110,6 @@ enum Command { Coordinator { port: Option }, } -enum Event { - CtrlC, -} - #[derive(Debug, clap::Args)] pub struct CommandNew { #[clap(long, value_enum, default_value_t = Kind::Dataflow)] @@ -169,8 +161,6 @@ fn run() -> eyre::Result<()> { } }; - let ctrlc_events = set_up_ctrlc_handler()?; - match args.command { Command::Check { dataflow } => match dataflow { Some(dataflow) => { @@ -265,13 +255,7 @@ fn run() -> eyre::Result<()> { .build() .context("tokio runtime failed")?; rt.block_on(async { - let (_, task) = dora_coordinator::start( - port, - ctrlc_events.map(|event| match event { - Event::CtrlC => dora_coordinator::Event::CtrlC, - }), - ) - .await?; + let (_, task) = dora_coordinator::start(port).await?; task.await }) .context("failed to run dora-coordinator")? @@ -300,15 +284,6 @@ fn run() -> eyre::Result<()> { (localhost, DORA_COORDINATOR_PORT_DEFAULT).into() }), machine_id.unwrap_or_default(), - ctrlc_events.map(|event| { - let clock = HLC::default(); - match event { - Event::CtrlC => Timestamped { - inner: dora_daemon::Event::CtrlC, - timestamp: clock.new_timestamp(), - }, - } - }), ) .await } @@ -439,25 +414,3 @@ fn query_running_dataflows( fn connect_to_coordinator() -> std::io::Result> { TcpLayer::new().connect(control_socket_addr()) } - -fn set_up_ctrlc_handler() -> Result, eyre::ErrReport> { - let (ctrlc_tx, ctrlc_rx) = mpsc::channel(1); - - let mut ctrlc_sent = false; - ctrlc::set_handler(move || { - if ctrlc_sent { - tracing::warn!("received second ctrlc signal -> aborting immediately"); - std::process::abort(); - } else { - tracing::info!("received ctrlc signal"); - if ctrlc_tx.blocking_send(Event::CtrlC).is_err() { - tracing::error!("failed to report ctrl-c event to dora-coordinator"); - } - - ctrlc_sent = true; - } - }) - .wrap_err("failed to set ctrl-c handler")?; - - Ok(ReceiverStream::new(ctrlc_rx)) -} diff --git a/binaries/coordinator/Cargo.toml b/binaries/coordinator/Cargo.toml index a1d30cd5..4c6b6887 100644 --- a/binaries/coordinator/Cargo.toml +++ b/binaries/coordinator/Cargo.toml @@ -24,3 +24,4 @@ dora-tracing = { workspace = true, optional = true } futures-concurrency = "7.1.0" serde_json = "1.0.86" names = "0.14.0" +ctrlc = "3.2.5" diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index f167bba2..c2e1a383 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -40,7 +40,6 @@ mod tcp_utils; pub async fn start( port: Option, - external_events: impl Stream + Unpin, ) -> Result<(u16, impl Future>), eyre::ErrReport> { let port = port.unwrap_or(DORA_COORDINATOR_PORT_DEFAULT); let listener = listener::create_listener(port).await?; @@ -49,8 +48,12 @@ pub async fn start( .wrap_err("failed to get local addr of listener")? .port(); let mut tasks = FuturesUnordered::new(); + + // Setup ctrl-c handler + let ctrlc_events = set_up_ctrlc_handler()?; + let future = async move { - start_inner(listener, &tasks, external_events).await?; + start_inner(listener, &tasks, ctrlc_events).await?; tracing::debug!("coordinator main loop finished, waiting on spawned tasks"); while let Some(join_result) = tasks.next().await { @@ -904,3 +907,25 @@ pub enum DaemonEvent { listen_socket: SocketAddr, }, } + +fn set_up_ctrlc_handler() -> Result, eyre::ErrReport> { + let (ctrlc_tx, ctrlc_rx) = mpsc::channel(1); + + let mut ctrlc_sent = false; + ctrlc::set_handler(move || { + if ctrlc_sent { + tracing::warn!("received second ctrlc signal -> aborting immediately"); + std::process::abort(); + } else { + tracing::info!("received ctrlc signal"); + if ctrlc_tx.blocking_send(Event::CtrlC).is_err() { + tracing::error!("failed to report ctrl-c event to dora-coordinator"); + } + + ctrlc_sent = true; + } + }) + .wrap_err("failed to set ctrl-c handler")?; + + Ok(ReceiverStream::new(ctrlc_rx)) +} diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index 6f6e9154..3577ed7d 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -34,3 +34,4 @@ shared-memory-server = { workspace = true } bincode = "1.3.3" async-trait = "0.1.64" aligned-vec = "0.5.0" +ctrlc = "3.2.5" diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 3fb9781c..9bf6d189 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -76,13 +76,11 @@ pub struct Daemon { } impl Daemon { - pub async fn run( - coordinator_addr: SocketAddr, - machine_id: String, - external_events: impl Stream> + Unpin, - ) -> eyre::Result<()> { + pub async fn run(coordinator_addr: SocketAddr, machine_id: String) -> eyre::Result<()> { let clock = Arc::new(HLC::default()); + let ctrlc_events = set_up_ctrlc_handler(clock.clone())?; + // spawn listen loop let (events_tx, events_rx) = flume::bounded(10); let listen_socket = @@ -108,7 +106,7 @@ impl Daemon { ); Self::run_general( - (coordinator_events, external_events, daemon_events).merge(), + (coordinator_events, ctrlc_events, daemon_events).merge(), Some(coordinator_addr), machine_id, None, @@ -1515,3 +1513,33 @@ fn send_with_timestamp( timestamp: clock.new_timestamp(), }) } + +fn set_up_ctrlc_handler( + clock: Arc, +) -> Result>, eyre::ErrReport> { + let (ctrlc_tx, ctrlc_rx) = mpsc::channel(1); + + let mut ctrlc_sent = false; + ctrlc::set_handler(move || { + if ctrlc_sent { + tracing::warn!("received second ctrlc signal -> aborting immediately"); + std::process::abort(); + } else { + tracing::info!("received ctrlc signal"); + if ctrlc_tx + .blocking_send(Timestamped { + inner: Event::CtrlC, + timestamp: clock.new_timestamp(), + }) + .is_err() + { + tracing::error!("failed to report ctrl-c event to dora-coordinator"); + } + + ctrlc_sent = true; + } + }) + .wrap_err("failed to set ctrl-c handler")?; + + Ok(ReceiverStream::new(ctrlc_rx)) +} From 054ae5afcc54c489d8331a2a42c82a7e852806a1 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 10:59:38 +0100 Subject: [PATCH 063/104] Fix multiple daemon example --- binaries/cli/src/main.rs | 4 +++- binaries/coordinator/src/lib.rs | 3 ++- examples/multiple-daemons/run.rs | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index bce1c1b0..404d2794 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -3,6 +3,7 @@ use std::{net::Ipv4Addr, path::PathBuf}; use attach::attach_dataflow; use clap::Parser; use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection}; +use dora_coordinator::Event; use dora_core::{ descriptor::Descriptor, topics::{ @@ -255,7 +256,8 @@ fn run() -> eyre::Result<()> { .build() .context("tokio runtime failed")?; rt.block_on(async { - let (_, task) = dora_coordinator::start(port).await?; + let (_, task) = + dora_coordinator::start(port, futures::stream::empty::()).await?; task.await }) .context("failed to run dora-coordinator")? diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index c2e1a383..28484db4 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -40,6 +40,7 @@ mod tcp_utils; pub async fn start( port: Option, + external_events: impl Stream + Unpin, ) -> Result<(u16, impl Future>), eyre::ErrReport> { let port = port.unwrap_or(DORA_COORDINATOR_PORT_DEFAULT); let listener = listener::create_listener(port).await?; @@ -53,7 +54,7 @@ pub async fn start( let ctrlc_events = set_up_ctrlc_handler()?; let future = async move { - start_inner(listener, &tasks, ctrlc_events).await?; + start_inner(listener, &tasks, (ctrlc_events, external_events).merge()).await?; tracing::debug!("coordinator main loop finished, waiting on spawned tasks"); while let Some(join_result) = tasks.next().await { diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index c3b7dddd..198124e9 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -5,7 +5,7 @@ use dora_core::{ }; use dora_tracing::set_up_tracing; use eyre::{bail, Context}; -use futures::stream; + use std::{ collections::BTreeSet, net::{Ipv4Addr, SocketAddr}, @@ -37,8 +37,8 @@ async fn main() -> eyre::Result<()> { let (coordinator_port, coordinator) = dora_coordinator::start(Some(0), ReceiverStream::new(coordinator_events_rx)).await?; let coordinator_addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), coordinator_port); - let daemon_a = dora_daemon::Daemon::run(coordinator_addr, "A".into(), stream::empty()); - let daemon_b = dora_daemon::Daemon::run(coordinator_addr, "B".into(), stream::empty()); + let daemon_a = dora_daemon::Daemon::run(coordinator_addr, "A".into()); + let daemon_b = dora_daemon::Daemon::run(coordinator_addr, "B".into()); tracing::info!("Spawning coordinator and daemons"); let mut tasks = JoinSet::new(); From 56bf90956c12aca275b4a7028b1b5f3118d0cd19 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 14:18:50 +0100 Subject: [PATCH 064/104] Keep release as zip --- .github/workflows/release.yml | 51 +++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa1aa47a..f4fcbf30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,7 +81,15 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-cli" + run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + + - name: Create Archive (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + New-Item -Path archive -ItemType Directory + Copy-Item target/release/dora.exe -Destination archive/dora.exe + Compress-Archive -Path archive\* -DestinationPath archive.zip - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 @@ -89,7 +97,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: target/release/dora.exe + asset_path: archive.zip asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip asset_content_type: application/zip @@ -109,7 +117,16 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-cli" + run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + + - name: "Create Archive (Unix)" + if: runner.os == 'Linux' || runner.os == 'macOS' + run: | + mkdir archive + cp target/release/dora archive/dora + cd archive + zip -r ../archive.zip . + cd .. - name: "Upload release asset" uses: actions/upload-release-asset@v1.0.1 @@ -117,7 +134,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: target/release/dora + asset_path: archive.zip asset_name: dora-${{ github.ref_name }}-x86_64-${{ runner.os }}.zip asset_content_type: application/zip @@ -141,7 +158,16 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-cli + args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli -p dora-daemon + + - name: "Archive Linux ARM64" + if: runner.os == 'Linux' + run: | + mkdir archive_aarch64 + cp target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora + cd archive_aarch64 + zip -r ../archive_aarch64.zip . + cd .. - name: "Upload Linux ARM64 asset" if: runner.os == 'Linux' @@ -150,7 +176,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: target/aarch64-unknown-linux-gnu/release/dora archive_aarch64/dora + asset_path: archive_aarch64.zip asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip asset_content_type: application/zip @@ -176,7 +202,16 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --release --target aarch64-apple-darwin -p dora-cli + args: --release --target aarch64-apple-darwin -p dora-coordinator -p dora-cli -p dora-daemon + + - name: "Archive macOS ARM64" + if: runner.os == 'macOS' + run: | + mkdir archive_aarch64 + cp target/aarch64-apple-darwin/release/dora archive_aarch64/dora + cd archive_aarch64 + zip -r ../archive_aarch64.zip . + cd .. - name: "Upload macOS ARM64 asset" if: runner.os == 'macOS' @@ -185,6 +220,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: target/aarch64-apple-darwin/release/dora archive_aarch64/dora + asset_path: archive_aarch64.zip asset_name: dora-${{ github.ref_name }}-aarch64-${{ runner.os }}.zip asset_content_type: application/zip From 05f919e06cc098f61002fbd212f37fd7948dd899 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 15:15:21 +0100 Subject: [PATCH 065/104] Use cli to spawn daemon run-dataflow to avoid confused exe path --- examples/benchmark/run.rs | 17 ++++++++++++++++- examples/c++-dataflow/run.rs | 17 ++++++++++++++++- examples/c-dataflow/run.rs | 17 ++++++++++++++++- examples/cmake-dataflow/run.rs | 17 ++++++++++++++++- examples/multiple-daemons/run.rs | 23 +++++++++++++++++++++-- examples/python-dataflow/run.rs | 19 +++++++++++++++++-- examples/python-operator-dataflow/run.rs | 19 +++++++++++++++++-- examples/python-ros2-dataflow/run.rs | 17 ++++++++++++++++- examples/rust-dataflow-url/run.rs | 17 ++++++++++++++++- examples/rust-dataflow/run.rs | 17 ++++++++++++++++- examples/rust-ros2-dataflow/run.rs | 17 ++++++++++++++++- 11 files changed, 183 insertions(+), 14 deletions(-) diff --git a/examples/benchmark/run.rs b/examples/benchmark/run.rs index 16df626e..78e6d88b 100644 --- a/examples/benchmark/run.rs +++ b/examples/benchmark/run.rs @@ -13,7 +13,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml"); build_dataflow(dataflow).await?; - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } @@ -29,3 +29,18 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 5f17138f..6f966e19 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -112,7 +112,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml").to_owned(); build_package("dora-runtime").await?; - dora_daemon::Daemon::run_dataflow(&dataflow).await?; + run_dataflow(&dataflow).await?; Ok(()) } @@ -128,6 +128,21 @@ async fn build_package(package: &str) -> eyre::Result<()> { Ok(()) } +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} + async fn build_cxx_node( root: &Path, paths: &[&Path], diff --git a/examples/c-dataflow/run.rs b/examples/c-dataflow/run.rs index 4ebcfdb5..ad484edf 100644 --- a/examples/c-dataflow/run.rs +++ b/examples/c-dataflow/run.rs @@ -23,7 +23,7 @@ async fn main() -> eyre::Result<()> { build_c_operator(root).await?; let dataflow = Path::new("dataflow.yml").to_owned(); - dora_daemon::Daemon::run_dataflow(&dataflow).await?; + run_dataflow(&dataflow).await?; Ok(()) } @@ -39,6 +39,21 @@ async fn build_package(package: &str) -> eyre::Result<()> { Ok(()) } +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} + async fn build_c_node(root: &Path, name: &str, out_name: &str) -> eyre::Result<()> { let mut clang = tokio::process::Command::new("clang"); clang.arg(name); diff --git a/examples/cmake-dataflow/run.rs b/examples/cmake-dataflow/run.rs index 27cbc68c..30e3c9d1 100644 --- a/examples/cmake-dataflow/run.rs +++ b/examples/cmake-dataflow/run.rs @@ -40,7 +40,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml").to_owned(); build_package("dora-runtime").await?; - dora_daemon::Daemon::run_dataflow(&dataflow).await?; + run_dataflow(&dataflow).await?; Ok(()) } @@ -55,3 +55,18 @@ async fn build_package(package: &str) -> eyre::Result<()> { } Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index 198124e9..78c50d3d 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -37,8 +37,8 @@ async fn main() -> eyre::Result<()> { let (coordinator_port, coordinator) = dora_coordinator::start(Some(0), ReceiverStream::new(coordinator_events_rx)).await?; let coordinator_addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), coordinator_port); - let daemon_a = dora_daemon::Daemon::run(coordinator_addr, "A".into()); - let daemon_b = dora_daemon::Daemon::run(coordinator_addr, "B".into()); + let daemon_a = run_dataflow(coordinator_addr.to_string(), "A".into(), dataflow); + let daemon_b = run_dataflow(coordinator_addr.to_string(), "B".into(), dataflow); tracing::info!("Spawning coordinator and daemons"); let mut tasks = JoinSet::new(); @@ -196,3 +196,22 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } + +async fn run_dataflow(coordinator: String, machine_id: &str, dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow) + .arg("--machine-id") + .arg(machine_id) + .arg("--coordinator-addr") + .arg(coordinator); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index e194e062..a14b553f 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,7 +1,7 @@ use dora_core::{get_pip_path, get_python_path, run}; use dora_download::download_file; use dora_tracing::set_up_tracing; -use eyre::{ContextCompat, WrapErr}; +use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; #[tokio::main] @@ -81,7 +81,22 @@ async fn main() -> eyre::Result<()> { .context("Could not download weights.")?; let dataflow = Path::new("dataflow.yml"); - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index 1413042f..c10dfc9a 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,7 +1,7 @@ use dora_core::{get_pip_path, get_python_path, run}; use dora_download::download_file; use dora_tracing::set_up_tracing; -use eyre::{ContextCompat, WrapErr}; +use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; #[tokio::main] @@ -81,7 +81,22 @@ async fn main() -> eyre::Result<()> { .await .context("Could not download weights.")?; let dataflow = Path::new("dataflow.yml"); - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 65e14129..6a9435d9 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -74,7 +74,22 @@ async fn main() -> eyre::Result<()> { .context("maturin develop failed")?; let dataflow = Path::new("dataflow.yml"); - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/rust-dataflow-url/run.rs b/examples/rust-dataflow-url/run.rs index c7c3d847..6f511970 100644 --- a/examples/rust-dataflow-url/run.rs +++ b/examples/rust-dataflow-url/run.rs @@ -13,7 +13,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml"); build_dataflow(dataflow).await?; - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } @@ -29,3 +29,18 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/rust-dataflow/run.rs b/examples/rust-dataflow/run.rs index 82095d03..f5e035a5 100644 --- a/examples/rust-dataflow/run.rs +++ b/examples/rust-dataflow/run.rs @@ -13,7 +13,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml"); build_dataflow(dataflow).await?; - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } @@ -29,3 +29,18 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} diff --git a/examples/rust-ros2-dataflow/run.rs b/examples/rust-ros2-dataflow/run.rs index 5818e03e..a14dce48 100644 --- a/examples/rust-ros2-dataflow/run.rs +++ b/examples/rust-ros2-dataflow/run.rs @@ -13,7 +13,7 @@ async fn main() -> eyre::Result<()> { let dataflow = Path::new("dataflow.yml"); build_dataflow(dataflow).await?; - dora_daemon::Daemon::run_dataflow(dataflow).await?; + run_dataflow(dataflow).await?; Ok(()) } @@ -29,3 +29,18 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { }; Ok(()) } + +async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { + let cargo = std::env::var("CARGO").unwrap(); + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--") + .arg("daemon") + .arg("--run-dataflow") + .arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + Ok(()) +} From 7fddec266546de371021d82d2876b6d90ebfd8aa Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 17:02:02 +0100 Subject: [PATCH 066/104] Make cli faster by reruning c_operator_type compilation dependent on modification of its source code --- apis/c/operator/build.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apis/c/operator/build.rs b/apis/c/operator/build.rs index a9b4adfd..6d45fd00 100644 --- a/apis/c/operator/build.rs +++ b/apis/c/operator/build.rs @@ -2,10 +2,11 @@ use std::path::Path; fn main() { dora_operator_api_types::generate_headers(Path::new("operator_types.h")) - .expect("failed to create operator_api.h"); + .expect("failed to create operator_types.h"); // don't rebuild on changes (otherwise we rebuild on every run as we're // writing the `operator_types.h` file; cargo will still rerun this script // when the `dora_operator_api_types` crate changes) - println!("cargo:rerun-if-changed="); + println!("cargo:rerun-if-changed=../../rust/operator/types/src/lib.rs"); + println!("cargo:rerun-if-changed=operator_api.h"); } From 28be4a6663142af63e2ab530be26b7c1b931132e Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 17:03:30 +0100 Subject: [PATCH 067/104] Fix example by using the right daemon command --- binaries/cli/src/main.rs | 22 ++++++++++++---------- binaries/daemon/src/lib.rs | 10 +++++++--- examples/multiple-daemons/run.rs | 11 +++++------ examples/python-ros2-dataflow/run.rs | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index 404d2794..cc71e198 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -256,7 +256,7 @@ fn run() -> eyre::Result<()> { .build() .context("tokio runtime failed")?; rt.block_on(async { - let (_, task) = + let (_port, task) = dora_coordinator::start(port, futures::stream::empty::()).await?; task.await }) @@ -276,19 +276,21 @@ fn run() -> eyre::Result<()> { Some(dataflow_path) => { tracing::info!("Starting dataflow `{}`", dataflow_path.display()); - Daemon::run_dataflow(&dataflow_path).await - } - None => { - Daemon::run( - coordinator_addr.unwrap_or_else(|| { - tracing::info!("Starting in local mode"); - let localhost = Ipv4Addr::new(127, 0, 0, 1); - (localhost, DORA_COORDINATOR_PORT_DEFAULT).into() - }), + Daemon::run_dataflow( + &dataflow_path, + coordinator_addr, machine_id.unwrap_or_default(), ) .await } + None => { + let addr = coordinator_addr.unwrap_or_else(|| { + tracing::info!("Starting in local mode"); + let localhost = Ipv4Addr::new(127, 0, 0, 1); + (localhost, DORA_COORDINATOR_PORT_DEFAULT).into() + }); + Daemon::run(addr, machine_id.unwrap_or_default()).await + } } }) .context("failed to run dora-daemon")? diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 9bf6d189..f5a2b8a5 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -116,7 +116,11 @@ impl Daemon { .map(|_| ()) } - pub async fn run_dataflow(dataflow_path: &Path) -> eyre::Result<()> { + pub async fn run_dataflow( + dataflow_path: &Path, + coordinator_addr: Option, + machine_id: String, + ) -> eyre::Result<()> { let working_dir = dataflow_path .canonicalize() .context("failed to canoncialize dataflow path")? @@ -156,8 +160,8 @@ impl Daemon { }); let run_result = Self::run_general( Box::pin(coordinator_events), - None, - "".into(), + coordinator_addr, + machine_id, Some(exit_when_done), clock, ); diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index 78c50d3d..d8fb13f6 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -35,10 +35,10 @@ async fn main() -> eyre::Result<()> { let (coordinator_events_tx, coordinator_events_rx) = mpsc::channel(1); let (coordinator_port, coordinator) = - dora_coordinator::start(Some(0), ReceiverStream::new(coordinator_events_rx)).await?; + dora_coordinator::start(None, ReceiverStream::new(coordinator_events_rx)).await?; let coordinator_addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), coordinator_port); - let daemon_a = run_dataflow(coordinator_addr.to_string(), "A".into(), dataflow); - let daemon_b = run_dataflow(coordinator_addr.to_string(), "B".into(), dataflow); + let daemon_a = run_daemon(coordinator_addr.to_string(), "A".into()); + let daemon_b = run_daemon(coordinator_addr.to_string(), "B".into()); tracing::info!("Spawning coordinator and daemons"); let mut tasks = JoinSet::new(); @@ -46,6 +46,7 @@ async fn main() -> eyre::Result<()> { tasks.spawn(daemon_a); tasks.spawn(daemon_b); + std::thread::sleep(Duration::from_secs(20)); // wait until both daemons are connected tracing::info!("waiting until daemons are connected to coordinator"); let mut retries = 0; @@ -197,15 +198,13 @@ async fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { Ok(()) } -async fn run_dataflow(coordinator: String, machine_id: &str, dataflow: &Path) -> eyre::Result<()> { +async fn run_daemon(coordinator: String, machine_id: &str) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); cmd.arg("--") .arg("daemon") - .arg("--run-dataflow") - .arg(dataflow) .arg("--machine-id") .arg(machine_id) .arg("--coordinator-addr") diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 6a9435d9..8eecdf64 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,6 +1,6 @@ use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; -use eyre::{ContextCompat, WrapErr}; +use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; #[tokio::main] From 5c16c59b6f09fed1034f6a2003d7956b7b180e12 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 23:36:11 +0100 Subject: [PATCH 068/104] Respond to coordinator destroy immediately to avoid end of file error --- binaries/daemon/src/coordinator.rs | 23 +++++++++++++++++++++++ binaries/daemon/src/lib.rs | 3 +-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/binaries/daemon/src/coordinator.rs b/binaries/daemon/src/coordinator.rs index 30f3bd07..082dfc6b 100644 --- a/binaries/daemon/src/coordinator.rs +++ b/binaries/daemon/src/coordinator.rs @@ -81,6 +81,29 @@ pub async fn register( timestamp, } = event; let (reply_tx, reply_rx) = oneshot::channel(); + + // Respond to the coordinator's destroy event immediately as the daemon is about to destroy itself. + match event { + DaemonCoordinatorEvent::Destroy => { + tracing::info!("Received destroy event from dora-coordinator"); + let serialized = + match serde_json::to_vec(&DaemonCoordinatorReply::DestroyResult(Ok(()))) + .wrap_err("failed to serialize DaemonCoordinatorReply") + { + Ok(r) => r, + Err(err) => { + tracing::error!("{err:?}"); + continue; + } + }; + if let Err(err) = tcp_send(&mut stream, &serialized).await { + tracing::warn!("failed to send reply to coordinator: {err}"); + continue; + }; + } + _ => {} + } + match tx .send(Timestamped { inner: CoordinatorEvent { event, reply_tx }, diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index f5a2b8a5..51c36671 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -431,9 +431,8 @@ impl Daemon { } DaemonCoordinatorEvent::Destroy => { tracing::info!("received destroy command -> exiting"); - let reply = DaemonCoordinatorReply::DestroyResult(Ok(())); let _ = reply_tx - .send(Some(reply)) + .send(None) .map_err(|_| error!("could not send destroy reply from daemon to coordinator")); RunStatus::Exit } From 249b2925f8cdf9d6a61e0ff0f57844f84a51d048 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 18 Jan 2024 14:53:57 +0100 Subject: [PATCH 069/104] Revert "Respond to coordinator destroy immediately to avoid end of file error" This reverts commit 5c16c59b6f09fed1034f6a2003d7956b7b180e12. --- binaries/daemon/src/coordinator.rs | 23 ----------------------- binaries/daemon/src/lib.rs | 3 ++- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/binaries/daemon/src/coordinator.rs b/binaries/daemon/src/coordinator.rs index 082dfc6b..30f3bd07 100644 --- a/binaries/daemon/src/coordinator.rs +++ b/binaries/daemon/src/coordinator.rs @@ -81,29 +81,6 @@ pub async fn register( timestamp, } = event; let (reply_tx, reply_rx) = oneshot::channel(); - - // Respond to the coordinator's destroy event immediately as the daemon is about to destroy itself. - match event { - DaemonCoordinatorEvent::Destroy => { - tracing::info!("Received destroy event from dora-coordinator"); - let serialized = - match serde_json::to_vec(&DaemonCoordinatorReply::DestroyResult(Ok(()))) - .wrap_err("failed to serialize DaemonCoordinatorReply") - { - Ok(r) => r, - Err(err) => { - tracing::error!("{err:?}"); - continue; - } - }; - if let Err(err) = tcp_send(&mut stream, &serialized).await { - tracing::warn!("failed to send reply to coordinator: {err}"); - continue; - }; - } - _ => {} - } - match tx .send(Timestamped { inner: CoordinatorEvent { event, reply_tx }, diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 51c36671..f5a2b8a5 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -431,8 +431,9 @@ impl Daemon { } DaemonCoordinatorEvent::Destroy => { tracing::info!("received destroy command -> exiting"); + let reply = DaemonCoordinatorReply::DestroyResult(Ok(())); let _ = reply_tx - .send(None) + .send(Some(reply)) .map_err(|_| error!("could not send destroy reply from daemon to coordinator")); RunStatus::Exit } From cb69a033f02697c223c7a31e2b4aa0d9b6117748 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 18 Jan 2024 15:11:06 +0100 Subject: [PATCH 070/104] Wait until `DestroyResult` is sent before exiting dora-daemon Tokio background tasks are canceled when the program exits, so we need some additional synchronization to ensure that the `DestroyResult` is actually sent out. --- binaries/coordinator/src/lib.rs | 2 +- binaries/daemon/src/coordinator.rs | 6 ++++++ binaries/daemon/src/lib.rs | 10 +++++++++- libraries/core/src/daemon_messages.rs | 6 +++++- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs index 28484db4..3d6be47d 100644 --- a/binaries/coordinator/src/lib.rs +++ b/binaries/coordinator/src/lib.rs @@ -852,7 +852,7 @@ async fn destroy_daemons( match serde_json::from_slice(&reply_raw) .wrap_err("failed to deserialize destroy reply from daemon")? { - DaemonCoordinatorReply::DestroyResult(result) => result + DaemonCoordinatorReply::DestroyResult { result, .. } => result .map_err(|e| eyre!(e)) .wrap_err("failed to destroy dataflow")?, other => bail!("unexpected reply after sending `destroy`: {other:?}"), diff --git a/binaries/daemon/src/coordinator.rs b/binaries/daemon/src/coordinator.rs index 30f3bd07..27ae8e49 100644 --- a/binaries/daemon/src/coordinator.rs +++ b/binaries/daemon/src/coordinator.rs @@ -113,6 +113,12 @@ pub async fn register( tracing::warn!("failed to send reply to coordinator: {err}"); continue; }; + if let DaemonCoordinatorReply::DestroyResult { notify, .. } = reply { + if let Some(notify) = notify { + let _ = notify.send(()); + } + break; + } } } }); diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index f5a2b8a5..9207e47d 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -431,10 +431,18 @@ impl Daemon { } DaemonCoordinatorEvent::Destroy => { tracing::info!("received destroy command -> exiting"); - let reply = DaemonCoordinatorReply::DestroyResult(Ok(())); + let (notify_tx, notify_rx) = oneshot::channel(); + let reply = DaemonCoordinatorReply::DestroyResult { + result: Ok(()), + notify: Some(notify_tx), + }; let _ = reply_tx .send(Some(reply)) .map_err(|_| error!("could not send destroy reply from daemon to coordinator")); + // wait until the reply is sent out + if notify_rx.await.is_err() { + tracing::warn!("no confirmation received for DestroyReply"); + } RunStatus::Exit } DaemonCoordinatorEvent::Heartbeat => { diff --git a/libraries/core/src/daemon_messages.rs b/libraries/core/src/daemon_messages.rs index a9a6bef8..623ea9c0 100644 --- a/libraries/core/src/daemon_messages.rs +++ b/libraries/core/src/daemon_messages.rs @@ -247,7 +247,11 @@ pub enum DaemonCoordinatorReply { SpawnResult(Result<(), String>), ReloadResult(Result<(), String>), StopResult(Result<(), String>), - DestroyResult(Result<(), String>), + DestroyResult { + result: Result<(), String>, + #[serde(skip)] + notify: Option>, + }, Logs(Result, String>), } From e701eb418230162ffa3597dc6f8c51a58863c616 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 18 Jan 2024 16:04:58 +0100 Subject: [PATCH 071/104] Fix: enable sync module of tokio --- libraries/core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index 72529189..25263f65 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -18,5 +18,5 @@ uuid = { version = "1.2.1", features = ["serde"] } dora-message = { workspace = true } tracing = "0.1" serde-with-expand-env = "1.1.0" -tokio = { version = "1.24.1", features = ["fs", "process"] } +tokio = { version = "1.24.1", features = ["fs", "process", "sync"] } aligned-vec = { version = "0.5.0", features = ["serde"] } From 7b37ac7218e5ff6bde3ee63a475d3278fda98903 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 13:13:00 +0100 Subject: [PATCH 072/104] Remove typo in release CI --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4fcbf30..43c31bdc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,7 +81,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + run: "cargo build --release -p dora-cli" - name: Create Archive (Windows) if: runner.os == 'Windows' @@ -117,7 +117,7 @@ jobs: - name: "Build binaries" timeout-minutes: 60 - run: "cargo build --release -p dora-coordinator -p dora-cli -p dora-daemon" + run: "cargo build --release -p dora-cli" - name: "Create Archive (Unix)" if: runner.os == 'Linux' || runner.os == 'macOS' @@ -158,7 +158,7 @@ jobs: with: use-cross: true command: build - args: --release --target aarch64-unknown-linux-gnu -p dora-coordinator -p dora-cli -p dora-daemon + args: --release --target aarch64-unknown-linux-gnu -p dora-cli - name: "Archive Linux ARM64" if: runner.os == 'Linux' From 80a79aa3a5d2cd1e90b6853a8b6ac4fa2f7d84f8 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 13:16:25 +0100 Subject: [PATCH 073/104] make `build.rs` faster by rerun only if `build.rs` changes --- apis/c/operator/build.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apis/c/operator/build.rs b/apis/c/operator/build.rs index 6d45fd00..93414639 100644 --- a/apis/c/operator/build.rs +++ b/apis/c/operator/build.rs @@ -7,6 +7,5 @@ fn main() { // don't rebuild on changes (otherwise we rebuild on every run as we're // writing the `operator_types.h` file; cargo will still rerun this script // when the `dora_operator_api_types` crate changes) - println!("cargo:rerun-if-changed=../../rust/operator/types/src/lib.rs"); - println!("cargo:rerun-if-changed=operator_api.h"); + println!("cargo:rerun-if-changed=build.rs"); } From 81154c4b9f6e4f216be32c1a2fcf3d2f213d9d73 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 13:20:57 +0100 Subject: [PATCH 074/104] Remove top-level unused dependencies --- Cargo.lock | 3 --- Cargo.toml | 3 --- 2 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da5514ef..af024277 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1507,10 +1507,8 @@ dependencies = [ name = "dora-examples" version = "0.0.0" dependencies = [ - "clap 4.4.6", "dora-coordinator", "dora-core", - "dora-daemon", "dora-download", "dora-tracing", "dunce", @@ -1520,7 +1518,6 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "tracing-subscriber", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 4d13bf32..4d533ed6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,6 @@ ros2-examples = [] [dev-dependencies] eyre = "0.6.8" tokio = "1.24.2" -dora-daemon = { workspace = true } dora-coordinator = { workspace = true } dora-core = { workspace = true } dora-tracing = { workspace = true } @@ -90,10 +89,8 @@ dunce = "1.0.2" serde_yaml = "0.8.23" uuid = { version = "1.2.1", features = ["v4", "serde"] } tracing = "0.1.36" -tracing-subscriber = "0.3.15" futures = "0.3.25" tokio-stream = "0.1.11" -clap = { version = "4.0.3", features = ["derive"] } [[example]] name = "c-dataflow" From eadf6b35577e47a0ebed94432064e24703a5936a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 14:18:45 +0100 Subject: [PATCH 075/104] Remove sleep in multiple daemons as compiling dora cli is fast enough --- examples/multiple-daemons/run.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs index d8fb13f6..4b1417cc 100644 --- a/examples/multiple-daemons/run.rs +++ b/examples/multiple-daemons/run.rs @@ -46,8 +46,6 @@ async fn main() -> eyre::Result<()> { tasks.spawn(daemon_a); tasks.spawn(daemon_b); - std::thread::sleep(Duration::from_secs(20)); - // wait until both daemons are connected tracing::info!("waiting until daemons are connected to coordinator"); let mut retries = 0; loop { From b79172c9bcfcf934a8a4c719aa93591026099106 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 14:22:29 +0100 Subject: [PATCH 076/104] Removing `coordinator_addr` for `run_dataflow` command --- binaries/cli/src/main.rs | 13 +++++++------ binaries/daemon/src/lib.rs | 10 +++------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index cc71e198..ab6383bf 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -275,13 +275,14 @@ fn run() -> eyre::Result<()> { match run_dataflow { Some(dataflow_path) => { tracing::info!("Starting dataflow `{}`", dataflow_path.display()); + if let Some(coordinator_addr) = coordinator_addr { + tracing::info!( + "Not using coordinator addr {} as `run_dataflow` is for local dataflow only. Please use the `start` command for remote coordinator", + coordinator_addr + ); + } - Daemon::run_dataflow( - &dataflow_path, - coordinator_addr, - machine_id.unwrap_or_default(), - ) - .await + Daemon::run_dataflow(&dataflow_path).await } None => { let addr = coordinator_addr.unwrap_or_else(|| { diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index 9207e47d..1515d343 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -116,11 +116,7 @@ impl Daemon { .map(|_| ()) } - pub async fn run_dataflow( - dataflow_path: &Path, - coordinator_addr: Option, - machine_id: String, - ) -> eyre::Result<()> { + pub async fn run_dataflow(dataflow_path: &Path) -> eyre::Result<()> { let working_dir = dataflow_path .canonicalize() .context("failed to canoncialize dataflow path")? @@ -160,8 +156,8 @@ impl Daemon { }); let run_result = Self::run_general( Box::pin(coordinator_events), - coordinator_addr, - machine_id, + None, + "".to_string(), Some(exit_when_done), clock, ); From 64fece9c47bdfe1a67d0ebfefe7022f484f4335d Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 22:14:16 +0100 Subject: [PATCH 077/104] Generate the right size of default value for array --- .../ros2-bridge/python/src/typed/mod.rs | 92 ++++++++++--------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index 150897fd..7fc4ddcd 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -1,10 +1,10 @@ use arrow::{ array::{ make_array, Array, ArrayData, BooleanArray, Float32Array, Float64Array, Int16Array, - Int32Array, Int64Array, Int8Array, StringArray, StructArray, UInt16Array, UInt32Array, - UInt64Array, UInt8Array, + Int32Array, Int64Array, Int8Array, ListArray, StringArray, StructArray, UInt16Array, + UInt32Array, UInt64Array, UInt8Array, }, - buffer::Buffer, + buffer::{OffsetBuffer, ScalarBuffer}, compute::concat, datatypes::{DataType, Field}, }; @@ -13,7 +13,7 @@ use dora_ros2_bridge_msg_gen::types::{ MemberType, Message, }; use eyre::{Context, ContextCompat, Result}; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, vec}; pub use serialize::TypedValue; @@ -40,7 +40,10 @@ pub fn for_message( .members .iter() .map(|m| { - let default = make_array(default_for_member(m, package_name, messages)?); + let default = make_array( + default_for_member(m, package_name, messages) + .context(format!("Could not create default value for {:#?}", m))?, + ); Result::<_, eyre::Report>::Ok(( Arc::new(Field::new( m.name.clone(), @@ -50,8 +53,8 @@ pub fn for_message( default, )) }) - .collect::>()?; - + .collect::>() + .context("Could not build default value")?; let default_struct: StructArray = default_struct_vec.into(); Ok(TypeInfo { @@ -77,28 +80,36 @@ pub fn default_for_member( Some(_) => eyre::bail!( "there should be only a single default value for non-sequence types" ), - None => default_for_nestable_type(t, package_name, messages)?, + None => default_for_nestable_type(t, package_name, messages, 1)?, }, NestableType::NamedType(_) => { if m.default.is_some() { eyre::bail!("default values for nested types are not supported") } else { - default_for_nestable_type(t, package_name, messages)? + default_for_nestable_type(t, package_name, messages, 1)? } } NestableType::NamespacedType(_) => { - default_for_nestable_type(t, package_name, messages)? + default_for_nestable_type(t, package_name, messages, 1)? } }, - MemberType::Array(array) => { - list_default_values(m, &array.value_type, package_name, messages)? - } + MemberType::Array(array) => list_default_values( + m, + &array.value_type, + package_name, + messages, + Some(array.size), + )?, MemberType::Sequence(seq) => { - list_default_values(m, &seq.value_type, package_name, messages)? - } - MemberType::BoundedSequence(seq) => { - list_default_values(m, &seq.value_type, package_name, messages)? + list_default_values(m, &seq.value_type, package_name, messages, None)? } + MemberType::BoundedSequence(seq) => list_default_values( + m, + &seq.value_type, + package_name, + messages, + Some(seq.max_size), + )?, }; Ok(value) } @@ -107,24 +118,25 @@ fn default_for_nestable_type( t: &NestableType, package_name: &str, messages: &HashMap>, + size: usize, ) -> Result { let empty = HashMap::new(); let package_messages = messages.get(package_name).unwrap_or(&empty); let array = match t { NestableType::BasicType(t) => match t { - BasicType::I8 => Int8Array::from(vec![0]).into(), - BasicType::I16 => Int16Array::from(vec![0]).into(), - BasicType::I32 => Int32Array::from(vec![0]).into(), - BasicType::I64 => Int64Array::from(vec![0]).into(), - BasicType::U8 => UInt8Array::from(vec![0]).into(), - BasicType::U16 => UInt16Array::from(vec![0]).into(), - BasicType::U32 => UInt32Array::from(vec![0]).into(), - BasicType::U64 => UInt64Array::from(vec![0]).into(), - BasicType::F32 => Float32Array::from(vec![0.]).into(), - BasicType::F64 => Float64Array::from(vec![0.]).into(), + BasicType::I8 => Int8Array::from(vec![0; size]).into(), + BasicType::I16 => Int16Array::from(vec![0; size]).into(), + BasicType::I32 => Int32Array::from(vec![0; size]).into(), + BasicType::I64 => Int64Array::from(vec![0; size]).into(), + BasicType::U8 => UInt8Array::from(vec![0; size]).into(), + BasicType::U16 => UInt16Array::from(vec![0; size]).into(), + BasicType::U32 => UInt32Array::from(vec![0; size]).into(), + BasicType::U64 => UInt64Array::from(vec![0; size]).into(), + BasicType::F32 => Float32Array::from(vec![0.; size]).into(), + BasicType::F64 => Float64Array::from(vec![0.; size]).into(), BasicType::Char => StringArray::from(vec![""]).into(), - BasicType::Byte => UInt8Array::from(vec![0u8] as Vec).into(), - BasicType::Bool => BooleanArray::from(vec![false]).into(), + BasicType::Byte => UInt8Array::from(vec![0u8; size]).into(), + BasicType::Bool => BooleanArray::from(vec![false; size]).into(), }, NestableType::GenericString(_) => StringArray::from(vec![""]).into(), NestableType::NamedType(name) => { @@ -230,6 +242,7 @@ fn list_default_values( value_type: &NestableType, package_name: &str, messages: &HashMap>, + size: Option, ) -> Result { let defaults = match &m.default.as_deref() { Some([]) => eyre::bail!("empty default value not supported"), @@ -253,23 +266,18 @@ fn list_default_values( default_values.to_data() } None => { + let size = size.unwrap_or(1); let default_nested_type = - default_for_nestable_type(value_type, package_name, messages)?; - - let value_offsets = Buffer::from_slice_ref([0i64, 1]); + default_for_nestable_type(value_type, package_name, messages, size)?; + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, size as i32])); - let list_data_type = DataType::List(Arc::new(Field::new( - &m.name, + let field = Arc::new(Field::new( + "item", default_nested_type.data_type().clone(), true, - ))); - // Construct a list array from the above two - ArrayData::builder(list_data_type) - .len(1) - .add_buffer(value_offsets) - .add_child_data(default_nested_type) - .build() - .context("Failed to build default list value")? + )); + let list = ListArray::new(field, offsets, make_array(default_nested_type), None); + list.to_data() } }; From f436fafc0770c95485dffede09b7eb578b4389c9 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 15 Jan 2024 22:15:41 +0100 Subject: [PATCH 078/104] Serialize by looping over the array This should fix: https://github.com/dora-rs/dora-autoware/discussions/9 --- .../extensions/ros2-bridge/python/src/lib.rs | 5 +- .../ros2-bridge/python/src/typed/serialize.rs | 189 ++++++++++++------ 2 files changed, 136 insertions(+), 58 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 37ac68a6..6c8f79c2 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -6,7 +6,7 @@ use std::{ use ::dora_ros2_bridge::{ros2_client, rustdds}; use arrow::{ - array::ArrayData, + array::{make_array, ArrayData}, pyarrow::{FromPyArrow, ToPyArrow}, }; use dora_ros2_bridge_msg_gen::types::Message; @@ -52,6 +52,7 @@ impl Ros2Context { ament_prefix_path_parsed.split(':').map(Path::new).collect() } }; + let packages = dora_ros2_bridge_msg_gen::get_packages(&paths) .map_err(|err| eyre!(err)) .context("failed to parse ROS2 message types")?; @@ -209,7 +210,7 @@ impl Ros2Publisher { //// add type info to ensure correct serialization (e.g. struct types //// and map types need to be serialized differently) let typed_value = TypedValue { - value: &value, + value: &make_array(value), type_info: &self.type_info, }; diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs index 15e8f622..32129f45 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs @@ -1,4 +1,7 @@ -use arrow::array::ArrayData; +use arrow::array::make_array; +use arrow::array::Array; +use arrow::array::ArrayRef; +use arrow::array::AsArray; use arrow::array::Float32Array; use arrow::array::Float64Array; use arrow::array::Int16Array; @@ -20,7 +23,7 @@ use super::TypeInfo; #[derive(Debug, Clone, PartialEq)] pub struct TypedValue<'a> { - pub value: &'a ArrayData, + pub value: &'a ArrayRef, pub type_info: &'a TypeInfo, } @@ -31,81 +34,154 @@ impl serde::Serialize for TypedValue<'_> { { match &self.type_info.data_type { DataType::UInt8 => { - let uint_array: UInt8Array = self.value.clone().into(); - let number = uint_array.value(0); - serializer.serialize_u8(number) + let uint_array: &UInt8Array = self.value.as_primitive(); + if uint_array.len() == 1 { + let number = uint_array.value(0); + serializer.serialize_u8(number) + } else { + let mut s = serializer.serialize_seq(Some(uint_array.len()))?; + for value in uint_array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::UInt16 => { - let uint_array: UInt16Array = self.value.clone().into(); - let number = uint_array.value(0); - serializer.serialize_u16(number) + let uint_array: &UInt16Array = self.value.as_primitive(); + if uint_array.len() == 1 { + let number = uint_array.value(0); + serializer.serialize_u16(number) + } else { + let mut s = serializer.serialize_seq(Some(uint_array.len()))?; + for value in uint_array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::UInt32 => { - let uint_array: UInt32Array = self.value.clone().into(); - let number = uint_array.value(0); - serializer.serialize_u32(number) + let array: &UInt32Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_u32(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::UInt64 => { - let uint_array: UInt64Array = self.value.clone().into(); - let number = uint_array.value(0); - serializer.serialize_u64(number) + let array: &UInt64Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_u64(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Int8 => { - let int_array: Int8Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_i8(number) + let array: &Int8Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_i8(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Int16 => { - let int_array: Int16Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_i16(number) + let array: &Int16Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_i16(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Int32 => { - let int_array: Int32Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_i32(number) + let array: &Int32Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_i32(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Int64 => { - let int_array: Int64Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_i64(number) + let array: &Int64Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_i64(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Float32 => { - let int_array: Float32Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_f32(number) + let array: &Float32Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_f32(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Float64 => { - let int_array: Float64Array = self.value.clone().into(); - let number = int_array.value(0); - serializer.serialize_f64(number) + let array: &Float64Array = self.value.as_primitive(); + if array.len() == 1 { + let number = array.value(0); + serializer.serialize_f64(number) + } else { + let mut s = serializer.serialize_seq(Some(array.len()))?; + for value in array.iter() { + s.serialize_element(&value)?; + } + s.end() + } } DataType::Utf8 => { - let int_array: StringArray = self.value.clone().into(); + let int_array: &StringArray = self.value.as_string(); let string = int_array.value(0); serializer.serialize_str(string) } DataType::List(field) => { - let list_array: ListArray = self.value.clone().into(); - let values = list_array.values(); - let mut s = serializer.serialize_seq(Some(values.len()))?; + let list_array: &ListArray = self.value.as_list(); + // let values: &UInt16Array = list_array.values().as_primitive(); + let mut s = serializer.serialize_seq(Some(list_array.len()))?; for value in list_array.iter() { - let value = match value { - Some(value) => value.to_data(), - None => { - return Err(serde::ser::Error::custom( - "Value in ListArray is null and not yet supported".to_string(), - )) - } - }; - - s.serialize_element(&TypedValue { - value: &value, - type_info: &TypeInfo { - data_type: field.data_type().clone(), - defaults: self.type_info.defaults.clone(), - }, - })?; + if let Some(value) = value { + s.serialize_element(&TypedValue { + value: &value, + type_info: &TypeInfo { + data_type: field.data_type().clone(), + defaults: self.type_info.defaults.clone(), + }, + })?; + } } s.end() } @@ -121,7 +197,7 @@ impl serde::Serialize for TypedValue<'_> { const DUMMY_STRUCT_NAME: &str = "struct"; const DUMMY_FIELD_NAME: &str = "field"; - let struct_array: StructArray = self.value.clone().into(); + let struct_array: &StructArray = self.value.as_struct(); let mut s = serializer.serialize_struct(DUMMY_STRUCT_NAME, fields.len())?; let defaults: StructArray = self.type_info.defaults.clone().into(); for field in fields.iter() { @@ -134,15 +210,16 @@ impl serde::Serialize for TypedValue<'_> { ))) } }; + let value = make_array(default.clone()); let field_value = match struct_array.column_by_name(field.name()) { - Some(value) => value.to_data(), - None => default.clone(), + Some(value) => value, + None => &value, }; s.serialize_field( DUMMY_FIELD_NAME, &TypedValue { - value: &field_value, + value: &field_value.clone(), type_info: &TypeInfo { data_type: field.data_type().clone(), defaults: default, From fe78982a262e42a517aaf99e3e87e0752f0ba5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:22:11 +0000 Subject: [PATCH 079/104] Bump h2 from 0.3.21 to 0.3.24 Bumps [h2](https://github.com/hyperium/h2) from 0.3.21 to 0.3.24. - [Release notes](https://github.com/hyperium/h2/releases) - [Changelog](https://github.com/hyperium/h2/blob/v0.3.24/CHANGELOG.md) - [Commits](https://github.com/hyperium/h2/compare/v0.3.21...v0.3.24) --- updated-dependencies: - dependency-name: h2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50aa206d..dfddd2fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2373,9 +2373,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -2383,7 +2383,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.0.2", "slab", "tokio", "tokio-util", From 4085ebc46be527b893c54b0e591a0761e0882760 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jan 2024 19:16:25 +0100 Subject: [PATCH 080/104] Make PrimitiveArray only serialize one element at a time --- binaries/runtime/src/operator/mod.rs | 1 + .../ros2-bridge/python/src/typed/serialize.rs | 273 ++++++++++-------- 2 files changed, 161 insertions(+), 113 deletions(-) diff --git a/binaries/runtime/src/operator/mod.rs b/binaries/runtime/src/operator/mod.rs index 474a54c5..b78fb939 100644 --- a/binaries/runtime/src/operator/mod.rs +++ b/binaries/runtime/src/operator/mod.rs @@ -13,6 +13,7 @@ pub mod channel; mod python; mod shared_lib; +#[allow(unused_variables)] pub fn run_operator( node_id: &NodeId, operator_definition: OperatorDefinition, diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs index 32129f45..5219489d 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs @@ -34,153 +34,200 @@ impl serde::Serialize for TypedValue<'_> { { match &self.type_info.data_type { DataType::UInt8 => { - let uint_array: &UInt8Array = self.value.as_primitive(); - if uint_array.len() == 1 { - let number = uint_array.value(0); - serializer.serialize_u8(number) - } else { - let mut s = serializer.serialize_seq(Some(uint_array.len()))?; - for value in uint_array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + let array: &UInt8Array = self.value.as_primitive(); + debug_assert!(array.len() == 1, "array length was: {}", array.len()); + let number = array.value(0); + serializer.serialize_u8(number) } DataType::UInt16 => { - let uint_array: &UInt16Array = self.value.as_primitive(); - if uint_array.len() == 1 { - let number = uint_array.value(0); - serializer.serialize_u16(number) - } else { - let mut s = serializer.serialize_seq(Some(uint_array.len()))?; - for value in uint_array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + let array: &UInt16Array = self.value.as_primitive(); + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_u16(number) } DataType::UInt32 => { let array: &UInt32Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_u32(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_u32(number) } DataType::UInt64 => { let array: &UInt64Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_u64(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_u64(number) } DataType::Int8 => { let array: &Int8Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_i8(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_i8(number) } DataType::Int16 => { let array: &Int16Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_i16(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_i16(number) } DataType::Int32 => { let array: &Int32Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_i32(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_i32(number) } DataType::Int64 => { let array: &Int64Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_i64(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1, "array was: {:#?}", array); + let number = array.value(0); + serializer.serialize_i64(number) } DataType::Float32 => { let array: &Float32Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_f32(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_f32(number) } DataType::Float64 => { let array: &Float64Array = self.value.as_primitive(); - if array.len() == 1 { - let number = array.value(0); - serializer.serialize_f64(number) - } else { - let mut s = serializer.serialize_seq(Some(array.len()))?; - for value in array.iter() { - s.serialize_element(&value)?; - } - s.end() - } + debug_assert!(array.len() == 1); + let number = array.value(0); + serializer.serialize_f64(number) } DataType::Utf8 => { let int_array: &StringArray = self.value.as_string(); let string = int_array.value(0); serializer.serialize_str(string) } - DataType::List(field) => { + DataType::List(_field) => { let list_array: &ListArray = self.value.as_list(); - // let values: &UInt16Array = list_array.values().as_primitive(); let mut s = serializer.serialize_seq(Some(list_array.len()))?; - for value in list_array.iter() { - if let Some(value) = value { - s.serialize_element(&TypedValue { - value: &value, - type_info: &TypeInfo { - data_type: field.data_type().clone(), - defaults: self.type_info.defaults.clone(), - }, - })?; + for root in list_array.iter() { + if let Some(values) = root { + match values.data_type() { + DataType::UInt8 => { + let values: &UInt8Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::UInt16 => { + let values: &UInt16Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::UInt32 => { + let values: &UInt32Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::UInt64 => { + let values: &UInt64Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Int8 => { + let values: &Int8Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Int16 => { + let values: &Int16Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Int32 => { + let values: &Int32Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Int64 => { + let values: &Int64Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Float32 => { + let values: &Float32Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Float64 => { + let values: &Float64Array = values.as_primitive(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Utf8 => { + let values: &StringArray = values.as_string(); + for value in values.iter() { + if let Some(value) = value { + s.serialize_element(&value)?; + } else { + todo!("Implement null management"); + } + } + } + DataType::Struct(_fields) => { + let list_array: ListArray = self.type_info.defaults.clone().into(); + s.serialize_element(&TypedValue { + value: &values, + type_info: &TypeInfo { + data_type: values.data_type().clone(), + defaults: list_array.value(0).to_data(), + }, + })?; + } + op => todo!("Implement additional type: {:?}", op), + } + } else { + todo!("Implement null management"); } } s.end() From 13273824e1fe5a27c98ab1a891a687ce7faba2af Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Sat, 20 Jan 2024 16:44:03 +0100 Subject: [PATCH 081/104] Rework python ROS2 (de)serialization using parsed ROS2 messages directly Use ROS2 types as type info to ensure that serialization type matches exactly. This is necessary because the types need to match exactly, otherwise the serialization or deserialization will result in invalid data. Using arrow schemas to specify the ROS2 message types does not work because not all ROS2 message types can be represented. For example, ROS2 serializes arrays with known length differently than sequences with dynamic length. --- .../extensions/ros2-bridge/python/src/lib.rs | 22 +- .../python/src/typed/deserialize.rs | 397 ------------------ .../python/src/typed/deserialize/array.rs | 17 + .../python/src/typed/deserialize/mod.rs | 292 +++++++++++++ .../python/src/typed/deserialize/primitive.rs | 155 +++++++ .../python/src/typed/deserialize/sequence.rs | 91 ++++ .../python/src/typed/deserialize/string.rs | 44 ++ .../ros2-bridge/python/src/typed/mod.rs | 294 +------------ .../ros2-bridge/python/src/typed/serialize.rs | 282 ------------- .../python/src/typed/serialize/array.rs | 186 ++++++++ .../python/src/typed/serialize/defaults.rs | 237 +++++++++++ .../python/src/typed/serialize/mod.rs | 181 ++++++++ .../python/src/typed/serialize/primitive.rs | 79 ++++ .../python/src/typed/serialize/sequence.rs | 152 +++++++ 14 files changed, 1463 insertions(+), 966 deletions(-) delete mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize/primitive.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/deserialize/string.rs delete mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize/defaults.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize/primitive.rs create mode 100644 libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 6c8f79c2..9d06bbaf 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, collections::HashMap, path::{Path, PathBuf}, sync::Arc, @@ -17,7 +18,7 @@ use pyo3::{ types::{PyDict, PyList, PyModule}, PyAny, PyObject, PyResult, Python, }; -use typed::{deserialize::TypedDeserializer, for_message, TypeInfo, TypedValue}; +use typed::{deserialize::StructDeserializer, TypeInfo, TypedValue}; pub mod qos; pub mod typed; @@ -112,10 +113,11 @@ impl Ros2Node { let topic = self .node .create_topic(&topic_name, message_type_name, &qos.into())?; - let type_info = - for_message(&self.messages, namespace_name, message_name).with_context(|| { - format!("failed to determine type info for message {namespace_name}/{message_name}") - })?; + let type_info = TypeInfo { + package_name: namespace_name.to_owned().into(), + message_name: message_name.to_owned().into(), + messages: self.messages.clone(), + }; Ok(Ros2Topic { topic, type_info }) } @@ -144,7 +146,7 @@ impl Ros2Node { .create_subscription(&topic.topic, qos.map(Into::into))?; Ok(Ros2Subscription { subscription: Some(subscription), - deserializer: TypedDeserializer::new(topic.type_info.clone()), + deserializer: StructDeserializer::new(Cow::Owned(topic.type_info.clone())), }) } } @@ -176,14 +178,14 @@ impl From for ros2_client::NodeOptions { #[non_exhaustive] pub struct Ros2Topic { topic: rustdds::Topic, - type_info: TypeInfo, + type_info: TypeInfo<'static>, } #[pyclass] #[non_exhaustive] pub struct Ros2Publisher { publisher: ros2_client::Publisher>, - type_info: TypeInfo, + type_info: TypeInfo<'static>, } #[pymethods] @@ -225,7 +227,7 @@ impl Ros2Publisher { #[pyclass] #[non_exhaustive] pub struct Ros2Subscription { - deserializer: TypedDeserializer, + deserializer: StructDeserializer<'static>, subscription: Option>, } @@ -264,7 +266,7 @@ impl Ros2Subscription { } pub struct Ros2SubscriptionStream { - deserializer: TypedDeserializer, + deserializer: StructDeserializer<'static>, subscription: ros2_client::Subscription, } diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize.rs deleted file mode 100644 index 30ba81fd..00000000 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize.rs +++ /dev/null @@ -1,397 +0,0 @@ -use super::TypeInfo; -use arrow::{ - array::{ - make_array, Array, ArrayData, BooleanBuilder, Float32Builder, Float64Builder, Int16Builder, - Int32Builder, Int64Builder, Int8Builder, ListArray, NullArray, StringBuilder, StructArray, - UInt16Builder, UInt32Builder, UInt64Builder, UInt8Builder, - }, - buffer::OffsetBuffer, - compute::concat, - datatypes::{DataType, Field, Fields}, -}; -use core::fmt; -use std::sync::Arc; -#[derive(Debug, Clone, PartialEq)] -pub struct TypedDeserializer { - type_info: TypeInfo, -} - -impl TypedDeserializer { - pub fn new(type_info: TypeInfo) -> Self { - Self { type_info } - } -} - -impl<'de> serde::de::DeserializeSeed<'de> for TypedDeserializer { - type Value = ArrayData; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let data_type = self.type_info.data_type; - let value = match data_type.clone() { - DataType::Struct(fields) => { - /// Serde requires that struct and field names are known at - /// compile time with a `'static` lifetime, which is not - /// possible in this case. Thus, we need to use dummy names - /// instead. - /// - /// The actual names do not really matter because - /// the CDR format of ROS2 does not encode struct or field - /// names. - const DUMMY_STRUCT_NAME: &str = "struct"; - const DUMMY_FIELDS: &[&str] = &[""; 100]; - - deserializer.deserialize_struct( - DUMMY_STRUCT_NAME, - &DUMMY_FIELDS[..fields.len()], - StructVisitor { - fields, - defaults: self.type_info.defaults, - }, - ) - } - DataType::List(field) => deserializer.deserialize_seq(ListVisitor { - field, - defaults: self.type_info.defaults, - }), - DataType::UInt8 => deserializer.deserialize_u8(PrimitiveValueVisitor), - DataType::UInt16 => deserializer.deserialize_u16(PrimitiveValueVisitor), - DataType::UInt32 => deserializer.deserialize_u32(PrimitiveValueVisitor), - DataType::UInt64 => deserializer.deserialize_u64(PrimitiveValueVisitor), - DataType::Int8 => deserializer.deserialize_i8(PrimitiveValueVisitor), - DataType::Int16 => deserializer.deserialize_i16(PrimitiveValueVisitor), - DataType::Int32 => deserializer.deserialize_i32(PrimitiveValueVisitor), - DataType::Int64 => deserializer.deserialize_i64(PrimitiveValueVisitor), - DataType::Float32 => deserializer.deserialize_f32(PrimitiveValueVisitor), - DataType::Float64 => deserializer.deserialize_f64(PrimitiveValueVisitor), - DataType::Utf8 => deserializer.deserialize_str(PrimitiveValueVisitor), - _ => todo!(), - }?; - - debug_assert!( - value.data_type() == &data_type, - "Datatype does not correspond to default data type.\n Expected: {:#?} \n but got: {:#?}, with value: {:#?}", data_type, value.data_type(), value - ); - - Ok(value) - } -} - -/// Based on https://docs.rs/serde_yaml/0.9.22/src/serde_yaml/value/de.rs.html#14-121 -struct PrimitiveValueVisitor; - -impl<'de> serde::de::Visitor<'de> for PrimitiveValueVisitor { - type Value = ArrayData; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a primitive value") - } - - fn visit_bool(self, b: bool) -> Result - where - E: serde::de::Error, - { - let mut array = BooleanBuilder::new(); - array.append_value(b); - Ok(array.finish().into()) - } - - fn visit_i8(self, u: i8) -> Result - where - E: serde::de::Error, - { - let mut array = Int8Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - - fn visit_i16(self, u: i16) -> Result - where - E: serde::de::Error, - { - let mut array = Int16Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - fn visit_i32(self, u: i32) -> Result - where - E: serde::de::Error, - { - let mut array = Int32Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - fn visit_i64(self, i: i64) -> Result - where - E: serde::de::Error, - { - let mut array = Int64Builder::new(); - array.append_value(i); - Ok(array.finish().into()) - } - - fn visit_u8(self, u: u8) -> Result - where - E: serde::de::Error, - { - let mut array = UInt8Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - fn visit_u16(self, u: u16) -> Result - where - E: serde::de::Error, - { - let mut array = UInt16Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - fn visit_u32(self, u: u32) -> Result - where - E: serde::de::Error, - { - let mut array = UInt32Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - fn visit_u64(self, u: u64) -> Result - where - E: serde::de::Error, - { - let mut array = UInt64Builder::new(); - array.append_value(u); - Ok(array.finish().into()) - } - - fn visit_f32(self, f: f32) -> Result - where - E: serde::de::Error, - { - let mut array = Float32Builder::new(); - array.append_value(f); - Ok(array.finish().into()) - } - - fn visit_f64(self, f: f64) -> Result - where - E: serde::de::Error, - { - let mut array = Float64Builder::new(); - array.append_value(f); - Ok(array.finish().into()) - } - - fn visit_str(self, s: &str) -> Result - where - E: serde::de::Error, - { - let mut array = StringBuilder::new(); - array.append_value(s); - Ok(array.finish().into()) - } - - fn visit_string(self, s: String) -> Result - where - E: serde::de::Error, - { - let mut array = StringBuilder::new(); - array.append_value(s); - Ok(array.finish().into()) - } - - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - let array = NullArray::new(0); - Ok(array.into()) - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - let array = NullArray::new(0); - Ok(array.into()) - } -} - -struct StructVisitor { - fields: Fields, - defaults: ArrayData, -} - -impl<'de> serde::de::Visitor<'de> for StructVisitor { - type Value = ArrayData; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a struct encoded as sequence") - } - - fn visit_seq(self, mut data: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut fields = vec![]; - let defaults: StructArray = self.defaults.clone().into(); - for field in self.fields.iter() { - let default = match defaults.column_by_name(field.name()) { - Some(value) => value.clone(), - None => { - return Err(serde::de::Error::custom(format!( - "missing field {} for deserialization", - &field.name() - ))) - } - }; - let value = match data.next_element_seed(TypedDeserializer { - type_info: TypeInfo { - data_type: field.data_type().clone(), - defaults: default.to_data(), - }, - })? { - Some(value) => make_array(value), - None => default, - }; - fields.push(( - // Recreate a new field as List(UInt8) can be converted to UInt8 - Arc::new(Field::new(field.name(), value.data_type().clone(), true)), - value, - )); - } - - let struct_array: StructArray = fields.into(); - - Ok(struct_array.into()) - } -} - -struct ListVisitor { - field: Arc, - defaults: ArrayData, -} - -impl<'de> serde::de::Visitor<'de> for ListVisitor { - type Value = ArrayData; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("an array encoded as sequence") - } - - fn visit_seq(self, mut data: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let data = match self.field.data_type().clone() { - DataType::UInt8 => { - let mut array = UInt8Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::UInt16 => { - let mut array = UInt16Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::UInt32 => { - let mut array = UInt32Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::UInt64 => { - let mut array = UInt64Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Int8 => { - let mut array = Int8Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Int16 => { - let mut array = Int16Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Int32 => { - let mut array = Int32Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Int64 => { - let mut array = Int64Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Float32 => { - let mut array = Float32Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Float64 => { - let mut array = Float64Builder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - DataType::Utf8 => { - let mut array = StringBuilder::new(); - while let Some(value) = data.next_element::()? { - array.append_value(value); - } - Ok(array.finish().into()) - } - _ => { - let mut buffer = vec![]; - while let Some(value) = data.next_element_seed(TypedDeserializer { - type_info: TypeInfo { - data_type: self.field.data_type().clone(), - defaults: self.defaults.clone(), - }, - })? { - let element = make_array(value); - buffer.push(element); - } - - concat( - buffer - .iter() - .map(|data| data.as_ref()) - .collect::>() - .as_slice(), - ) - .map(|op| op.to_data()) - } - }; - - if let Ok(values) = data { - let offsets = OffsetBuffer::new(vec![0, values.len() as i32].into()); - - let array = ListArray::new(self.field, offsets, make_array(values), None).to_data(); - Ok(array) - } else { - Ok(self.defaults) // TODO: Better handle deserialization error - } - } -} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs new file mode 100644 index 00000000..79030abe --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs @@ -0,0 +1,17 @@ +use arrow::array::ArrayData; +use dora_ros2_bridge_msg_gen::types::sequences; + +use super::sequence::SequenceVisitor; + +pub struct ArrayDeserializer<'a>(pub &'a sequences::Array); + +impl<'de> serde::de::DeserializeSeed<'de> for ArrayDeserializer<'_> { + type Value = ArrayData; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_tuple(self.0.size, SequenceVisitor(&self.0.value_type)) + } +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs new file mode 100644 index 00000000..93a94eee --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs @@ -0,0 +1,292 @@ +use super::{TypeInfo, DUMMY_STRUCT_NAME}; +use arrow::{ + array::{make_array, ArrayData, StructArray}, + datatypes::Field, +}; +use core::fmt; +use std::{borrow::Cow, collections::HashMap, fmt::Display, sync::Arc}; + +mod array; +mod primitive; +mod sequence; +mod string; + +#[derive(Debug, Clone)] +pub struct StructDeserializer<'a> { + type_info: Cow<'a, TypeInfo<'a>>, +} + +impl<'a> StructDeserializer<'a> { + pub fn new(type_info: Cow<'a, TypeInfo<'a>>) -> Self { + Self { type_info } + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for StructDeserializer<'_> { + type Value = ArrayData; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let empty = HashMap::new(); + let package_messages = self + .type_info + .messages + .get(self.type_info.package_name.as_ref()) + .unwrap_or(&empty); + let message = package_messages + .get(self.type_info.message_name.as_ref()) + .ok_or_else(|| { + error(format!( + "could not find message type {}::{}", + self.type_info.package_name, self.type_info.message_name + )) + })?; + + let visitor = StructVisitor { + type_info: self.type_info.as_ref(), + }; + deserializer.deserialize_tuple_struct(DUMMY_STRUCT_NAME, message.members.len(), visitor) + } +} + +struct StructVisitor<'a> { + type_info: &'a TypeInfo<'a>, +} + +impl<'a, 'de> serde::de::Visitor<'de> for StructVisitor<'a> { + type Value = ArrayData; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a struct encoded as sequence") + } + + fn visit_seq(self, mut data: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let empty = HashMap::new(); + let package_messages = self + .type_info + .messages + .get(self.type_info.package_name.as_ref()) + .unwrap_or(&empty); + let message = package_messages + .get(self.type_info.message_name.as_ref()) + .ok_or_else(|| { + error(format!( + "could not find message type {}::{}", + self.type_info.package_name, self.type_info.message_name + )) + })?; + + let mut fields = vec![]; + for member in &message.members { + let value = match &member.r#type { + dora_ros2_bridge_msg_gen::types::MemberType::NestableType(t) => match t { + dora_ros2_bridge_msg_gen::types::primitives::NestableType::BasicType(t) => { + data.next_element_seed(primitive::PrimitiveDeserializer(t))? + } + dora_ros2_bridge_msg_gen::types::primitives::NestableType::NamedType(name) => { + data.next_element_seed(StructDeserializer { + type_info: Cow::Owned(TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }), + })? + } + dora_ros2_bridge_msg_gen::types::primitives::NestableType::NamespacedType( + reference, + ) => { + if reference.namespace != "msg" { + return Err(error(format!( + "struct field {} references non-message type {reference:?}", + member.name + ))); + } + data.next_element_seed(StructDeserializer { + type_info: Cow::Owned(TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }), + })? + } + dora_ros2_bridge_msg_gen::types::primitives::NestableType::GenericString(t) => { + match t { + dora_ros2_bridge_msg_gen::types::primitives::GenericString::String | dora_ros2_bridge_msg_gen::types::primitives::GenericString::BoundedString(_)=> { + data.next_element_seed(string::StringDeserializer)? + }, + dora_ros2_bridge_msg_gen::types::primitives::GenericString::WString => todo!("deserialize WString"), + dora_ros2_bridge_msg_gen::types::primitives::GenericString::BoundedWString(_) => todo!("deserialize BoundedWString"), + } + } + }, + dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { + data.next_element_seed(array::ArrayDeserializer(a))? + }, + dora_ros2_bridge_msg_gen::types::MemberType::Sequence(s) => { + data.next_element_seed(sequence::SequenceDeserializer(&s.value_type))? + }, + dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(s) => { + data.next_element_seed(sequence::SequenceDeserializer(&s.value_type))? + }, + }; + + let value = value.ok_or_else(|| { + error(format!( + "struct member {} not present in message", + member.name + )) + })?; + + fields.push(( + // Recreate a new field as List(UInt8) can be converted to UInt8 + Arc::new(Field::new(&member.name, value.data_type().clone(), true)), + make_array(value), + )); + } + + let struct_array: StructArray = fields.into(); + + Ok(struct_array.into()) + } +} + +// struct ListVisitor { +// field: Arc, +// defaults: ArrayData, +// } + +// impl<'de> serde::de::Visitor<'de> for ListVisitor { +// type Value = ArrayData; + +// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { +// formatter.write_str("an array encoded as sequence") +// } + +// fn visit_seq(self, mut data: A) -> Result +// where +// A: serde::de::SeqAccess<'de>, +// { +// let data = match self.field.data_type().clone() { +// DataType::UInt8 => { +// let mut array = UInt8Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::UInt16 => { +// let mut array = UInt16Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::UInt32 => { +// let mut array = UInt32Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::UInt64 => { +// let mut array = UInt64Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Int8 => { +// let mut array = Int8Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Int16 => { +// let mut array = Int16Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Int32 => { +// let mut array = Int32Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Int64 => { +// let mut array = Int64Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Float32 => { +// let mut array = Float32Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Float64 => { +// let mut array = Float64Builder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// DataType::Utf8 => { +// let mut array = StringBuilder::new(); +// while let Some(value) = data.next_element::()? { +// array.append_value(value); +// } +// Ok(array.finish().into()) +// } +// _ => { +// let mut buffer = vec![]; +// while let Some(value) = data.next_element_seed(StructDeserializer { +// type_info: TypeInfo { +// data_type: self.field.data_type().clone(), +// defaults: self.defaults.clone(), +// }, +// })? { +// let element = make_array(value); +// buffer.push(element); +// } + +// concat( +// buffer +// .iter() +// .map(|data| data.as_ref()) +// .collect::>() +// .as_slice(), +// ) +// .map(|op| op.to_data()) +// } +// }; + +// if let Ok(values) = data { +// let offsets = OffsetBuffer::new(vec![0, values.len() as i32].into()); + +// let array = ListArray::new(self.field, offsets, make_array(values), None).to_data(); +// Ok(array) +// } else { +// Ok(self.defaults) // TODO: Better handle deserialization error +// } +// } +// } + +fn error(e: T) -> E +where + T: Display, + E: serde::de::Error, +{ + serde::de::Error::custom(e) +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/primitive.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/primitive.rs new file mode 100644 index 00000000..7f13b575 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/primitive.rs @@ -0,0 +1,155 @@ +use arrow::array::{ + ArrayData, BooleanBuilder, Float32Builder, Float64Builder, Int16Builder, Int32Builder, + Int64Builder, Int8Builder, NullArray, UInt16Builder, UInt32Builder, UInt64Builder, + UInt8Builder, +}; +use core::fmt; +use dora_ros2_bridge_msg_gen::types::primitives::BasicType; + +pub struct PrimitiveDeserializer<'a>(pub &'a BasicType); + +impl<'de> serde::de::DeserializeSeed<'de> for PrimitiveDeserializer<'_> { + type Value = ArrayData; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + match self.0 { + BasicType::I8 => deserializer.deserialize_i8(PrimitiveValueVisitor), + BasicType::I16 => deserializer.deserialize_i16(PrimitiveValueVisitor), + BasicType::I32 => deserializer.deserialize_i32(PrimitiveValueVisitor), + BasicType::I64 => deserializer.deserialize_i64(PrimitiveValueVisitor), + BasicType::U8 | BasicType::Char | BasicType::Byte => { + deserializer.deserialize_u8(PrimitiveValueVisitor) + } + BasicType::U16 => deserializer.deserialize_u16(PrimitiveValueVisitor), + BasicType::U32 => deserializer.deserialize_u32(PrimitiveValueVisitor), + BasicType::U64 => deserializer.deserialize_u64(PrimitiveValueVisitor), + BasicType::F32 => deserializer.deserialize_f32(PrimitiveValueVisitor), + BasicType::F64 => deserializer.deserialize_f64(PrimitiveValueVisitor), + BasicType::Bool => deserializer.deserialize_bool(PrimitiveValueVisitor), + } + } +} + +/// Based on https://docs.rs/serde_yaml/0.9.22/src/serde_yaml/value/de.rs.html#14-121 +struct PrimitiveValueVisitor; + +impl<'de> serde::de::Visitor<'de> for PrimitiveValueVisitor { + type Value = ArrayData; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a primitive value") + } + + fn visit_bool(self, b: bool) -> Result + where + E: serde::de::Error, + { + let mut array = BooleanBuilder::new(); + array.append_value(b); + Ok(array.finish().into()) + } + + fn visit_i8(self, u: i8) -> Result + where + E: serde::de::Error, + { + let mut array = Int8Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + + fn visit_i16(self, u: i16) -> Result + where + E: serde::de::Error, + { + let mut array = Int16Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + fn visit_i32(self, u: i32) -> Result + where + E: serde::de::Error, + { + let mut array = Int32Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + fn visit_i64(self, i: i64) -> Result + where + E: serde::de::Error, + { + let mut array = Int64Builder::new(); + array.append_value(i); + Ok(array.finish().into()) + } + + fn visit_u8(self, u: u8) -> Result + where + E: serde::de::Error, + { + let mut array = UInt8Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + fn visit_u16(self, u: u16) -> Result + where + E: serde::de::Error, + { + let mut array = UInt16Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + fn visit_u32(self, u: u32) -> Result + where + E: serde::de::Error, + { + let mut array = UInt32Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + fn visit_u64(self, u: u64) -> Result + where + E: serde::de::Error, + { + let mut array = UInt64Builder::new(); + array.append_value(u); + Ok(array.finish().into()) + } + + fn visit_f32(self, f: f32) -> Result + where + E: serde::de::Error, + { + let mut array = Float32Builder::new(); + array.append_value(f); + Ok(array.finish().into()) + } + + fn visit_f64(self, f: f64) -> Result + where + E: serde::de::Error, + { + let mut array = Float64Builder::new(); + array.append_value(f); + Ok(array.finish().into()) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + let array = NullArray::new(0); + Ok(array.into()) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + let array = NullArray::new(0); + Ok(array.into()) + } +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs new file mode 100644 index 00000000..47395012 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs @@ -0,0 +1,91 @@ +use arrow::{ + array::{ArrayData, BooleanBuilder, PrimitiveBuilder, StringBuilder}, + datatypes::{self, ArrowPrimitiveType}, +}; +use core::fmt; +use dora_ros2_bridge_msg_gen::types::primitives::{self, BasicType, NestableType}; +use serde::Deserialize; + +pub struct SequenceDeserializer<'a>(pub &'a NestableType); + +impl<'de> serde::de::DeserializeSeed<'de> for SequenceDeserializer<'_> { + type Value = ArrayData; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(SequenceVisitor(self.0)) + } +} + +pub struct SequenceVisitor<'a>(pub &'a NestableType); + +impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { + type Value = ArrayData; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a sequence") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + match &self.0 { + NestableType::BasicType(t) => match t { + BasicType::I8 => deserialize_primitive_seq::<_, datatypes::Int8Type>(seq), + BasicType::I16 => deserialize_primitive_seq::<_, datatypes::Int16Type>(seq), + BasicType::I32 => deserialize_primitive_seq::<_, datatypes::Int32Type>(seq), + BasicType::I64 => deserialize_primitive_seq::<_, datatypes::Int64Type>(seq), + BasicType::U8 | BasicType::Char | BasicType::Byte => { + deserialize_primitive_seq::<_, datatypes::UInt8Type>(seq) + } + BasicType::U16 => deserialize_primitive_seq::<_, datatypes::UInt16Type>(seq), + BasicType::U32 => deserialize_primitive_seq::<_, datatypes::UInt32Type>(seq), + BasicType::U64 => deserialize_primitive_seq::<_, datatypes::UInt64Type>(seq), + BasicType::F32 => deserialize_primitive_seq::<_, datatypes::Float32Type>(seq), + BasicType::F64 => deserialize_primitive_seq::<_, datatypes::Float64Type>(seq), + BasicType::Bool => { + let mut array = BooleanBuilder::new(); + while let Some(value) = seq.next_element()? { + array.append_value(value); + } + Ok(array.finish().into()) + } + }, + NestableType::NamedType(_) => todo!("deserialize sequence of NestableType::NamedType"), + NestableType::NamespacedType(_) => { + todo!("deserialize sequence of NestableType::NamedspacedType") + } + NestableType::GenericString(t) => match t { + primitives::GenericString::String | primitives::GenericString::BoundedString(_) => { + let mut array = StringBuilder::new(); + while let Some(value) = seq.next_element::()? { + array.append_value(value); + } + Ok(array.finish().into()) + } + primitives::GenericString::WString => todo!("deserialize sequence of WString"), + primitives::GenericString::BoundedWString(_) => { + todo!("deserialize sequence of BoundedWString") + } + }, + } + } +} + +fn deserialize_primitive_seq<'de, S, T>( + mut seq: S, +) -> Result>::Error> +where + S: serde::de::SeqAccess<'de>, + T: ArrowPrimitiveType, + T::Native: Deserialize<'de>, +{ + let mut array = PrimitiveBuilder::::new(); + while let Some(value) = seq.next_element::()? { + array.append_value(value); + } + Ok(array.finish().into()) +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/string.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/string.rs new file mode 100644 index 00000000..646ea38d --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/string.rs @@ -0,0 +1,44 @@ +use arrow::array::{ArrayData, StringBuilder}; +use core::fmt; + +pub struct StringDeserializer; + +impl<'de> serde::de::DeserializeSeed<'de> for StringDeserializer { + type Value = ArrayData; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(StringVisitor) + } +} + +/// Based on https://docs.rs/serde_yaml/0.9.22/src/serde_yaml/value/de.rs.html#14-121 +struct StringVisitor; + +impl<'de> serde::de::Visitor<'de> for StringVisitor { + type Value = ArrayData; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string value") + } + + fn visit_str(self, s: &str) -> Result + where + E: serde::de::Error, + { + let mut array = StringBuilder::new(); + array.append_value(s); + Ok(array.finish().into()) + } + + fn visit_string(self, s: String) -> Result + where + E: serde::de::Error, + { + let mut array = StringBuilder::new(); + array.append_value(s); + Ok(array.finish().into()) + } +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index 7fc4ddcd..b8b76893 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -1,285 +1,25 @@ -use arrow::{ - array::{ - make_array, Array, ArrayData, BooleanArray, Float32Array, Float64Array, Int16Array, - Int32Array, Int64Array, Int8Array, ListArray, StringArray, StructArray, UInt16Array, - UInt32Array, UInt64Array, UInt8Array, - }, - buffer::{OffsetBuffer, ScalarBuffer}, - compute::concat, - datatypes::{DataType, Field}, -}; -use dora_ros2_bridge_msg_gen::types::{ - primitives::{BasicType, NestableType}, - MemberType, Message, -}; -use eyre::{Context, ContextCompat, Result}; -use std::{collections::HashMap, sync::Arc, vec}; +use dora_ros2_bridge_msg_gen::types::Message; +use std::{borrow::Cow, collections::HashMap, sync::Arc}; pub use serialize::TypedValue; pub mod deserialize; pub mod serialize; -#[derive(Debug, Clone, PartialEq)] -pub struct TypeInfo { - data_type: DataType, - defaults: ArrayData, +#[derive(Debug, Clone)] +pub struct TypeInfo<'a> { + pub package_name: Cow<'a, str>, + pub message_name: Cow<'a, str>, + pub messages: Arc>>, } -pub fn for_message( - messages: &HashMap>, - package_name: &str, - message_name: &str, -) -> eyre::Result { - let empty = HashMap::new(); - let package_messages = messages.get(package_name).unwrap_or(&empty); - let message = package_messages - .get(message_name) - .context("unknown type name")?; - let default_struct_vec: Vec<(Arc, Arc)> = message - .members - .iter() - .map(|m| { - let default = make_array( - default_for_member(m, package_name, messages) - .context(format!("Could not create default value for {:#?}", m))?, - ); - Result::<_, eyre::Report>::Ok(( - Arc::new(Field::new( - m.name.clone(), - default.data_type().clone(), - true, - )), - default, - )) - }) - .collect::>() - .context("Could not build default value")?; - let default_struct: StructArray = default_struct_vec.into(); - - Ok(TypeInfo { - data_type: default_struct.data_type().clone(), - defaults: default_struct.into(), - }) -} - -pub fn default_for_member( - m: &dora_ros2_bridge_msg_gen::types::Member, - package_name: &str, - messages: &HashMap>, -) -> eyre::Result { - let value = match &m.r#type { - MemberType::NestableType(t) => match t { - NestableType::BasicType(_) | NestableType::GenericString(_) => match &m - .default - .as_deref() - { - Some([]) => eyre::bail!("empty default value not supported"), - Some([default]) => preset_default_for_basic_type(t, default) - .with_context(|| format!("failed to parse default value for `{}`", m.name))?, - Some(_) => eyre::bail!( - "there should be only a single default value for non-sequence types" - ), - None => default_for_nestable_type(t, package_name, messages, 1)?, - }, - NestableType::NamedType(_) => { - if m.default.is_some() { - eyre::bail!("default values for nested types are not supported") - } else { - default_for_nestable_type(t, package_name, messages, 1)? - } - } - NestableType::NamespacedType(_) => { - default_for_nestable_type(t, package_name, messages, 1)? - } - }, - MemberType::Array(array) => list_default_values( - m, - &array.value_type, - package_name, - messages, - Some(array.size), - )?, - MemberType::Sequence(seq) => { - list_default_values(m, &seq.value_type, package_name, messages, None)? - } - MemberType::BoundedSequence(seq) => list_default_values( - m, - &seq.value_type, - package_name, - messages, - Some(seq.max_size), - )?, - }; - Ok(value) -} - -fn default_for_nestable_type( - t: &NestableType, - package_name: &str, - messages: &HashMap>, - size: usize, -) -> Result { - let empty = HashMap::new(); - let package_messages = messages.get(package_name).unwrap_or(&empty); - let array = match t { - NestableType::BasicType(t) => match t { - BasicType::I8 => Int8Array::from(vec![0; size]).into(), - BasicType::I16 => Int16Array::from(vec![0; size]).into(), - BasicType::I32 => Int32Array::from(vec![0; size]).into(), - BasicType::I64 => Int64Array::from(vec![0; size]).into(), - BasicType::U8 => UInt8Array::from(vec![0; size]).into(), - BasicType::U16 => UInt16Array::from(vec![0; size]).into(), - BasicType::U32 => UInt32Array::from(vec![0; size]).into(), - BasicType::U64 => UInt64Array::from(vec![0; size]).into(), - BasicType::F32 => Float32Array::from(vec![0.; size]).into(), - BasicType::F64 => Float64Array::from(vec![0.; size]).into(), - BasicType::Char => StringArray::from(vec![""]).into(), - BasicType::Byte => UInt8Array::from(vec![0u8; size]).into(), - BasicType::Bool => BooleanArray::from(vec![false; size]).into(), - }, - NestableType::GenericString(_) => StringArray::from(vec![""]).into(), - NestableType::NamedType(name) => { - let referenced_message = package_messages - .get(&name.0) - .context("unknown referenced message")?; - - default_for_referenced_message(referenced_message, package_name, messages)? - } - NestableType::NamespacedType(t) => { - let referenced_package_messages = messages.get(&t.package).unwrap_or(&empty); - let referenced_message = referenced_package_messages - .get(&t.name) - .context("unknown referenced message")?; - default_for_referenced_message(referenced_message, &t.package, messages)? - } - }; - Ok(array) -} - -fn preset_default_for_basic_type(t: &NestableType, preset: &str) -> Result { - Ok(match t { - NestableType::BasicType(t) => match t { - BasicType::I8 => Int8Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::I16 => Int16Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::I32 => Int32Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::I64 => Int64Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::U8 => UInt8Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::U16 => UInt16Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::U32 => UInt32Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::U64 => UInt64Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::F32 => Float32Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::F64 => Float64Array::from(vec![preset - .parse::() - .context("Could not parse preset default value")?]) - .into(), - BasicType::Char => StringArray::from(vec![preset]).into(), - BasicType::Byte => UInt8Array::from(preset.as_bytes().to_owned()).into(), - BasicType::Bool => BooleanArray::from(vec![preset - .parse::() - .context("could not parse preset default value")?]) - .into(), - }, - NestableType::GenericString(_) => StringArray::from(vec![preset]).into(), - _ => todo!(), - }) -} - -fn default_for_referenced_message( - referenced_message: &Message, - package_name: &str, - messages: &HashMap>, -) -> eyre::Result { - let fields: Vec<(Arc, Arc)> = referenced_message - .members - .iter() - .map(|m| { - let default = default_for_member(m, package_name, messages)?; - Result::<_, eyre::Report>::Ok(( - Arc::new(Field::new( - m.name.clone(), - default.data_type().clone(), - true, - )), - make_array(default), - )) - }) - .collect::>()?; - - let struct_array: StructArray = fields.into(); - Ok(struct_array.into()) -} - -fn list_default_values( - m: &dora_ros2_bridge_msg_gen::types::Member, - value_type: &NestableType, - package_name: &str, - messages: &HashMap>, - size: Option, -) -> Result { - let defaults = match &m.default.as_deref() { - Some([]) => eyre::bail!("empty default value not supported"), - Some(defaults) => { - let raw_array: Vec> = defaults - .iter() - .map(|default| { - preset_default_for_basic_type(value_type, default) - .with_context(|| format!("failed to parse default value for `{}`", m.name)) - .map(make_array) - }) - .collect::>()?; - let default_values = concat( - raw_array - .iter() - .map(|data| data.as_ref()) - .collect::>() - .as_slice(), - ) - .context("Failed to concatenate default list value")?; - default_values.to_data() - } - None => { - let size = size.unwrap_or(1); - let default_nested_type = - default_for_nestable_type(value_type, package_name, messages, size)?; - let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, size as i32])); - - let field = Arc::new(Field::new( - "item", - default_nested_type.data_type().clone(), - true, - )); - let list = ListArray::new(field, offsets, make_array(default_nested_type), None); - list.to_data() - } - }; - - Ok(defaults) -} +/// Serde requires that struct and field names are known at +/// compile time with a `'static` lifetime, which is not +/// possible in this case. Thus, we need to use dummy names +/// instead. +/// +/// The actual names do not really matter because +/// the CDR format of ROS2 does not encode struct or field +/// names. +const DUMMY_STRUCT_NAME: &str = "struct"; +const DUMMY_FIELD_NAME: &str = "field"; diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs deleted file mode 100644 index 5219489d..00000000 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize.rs +++ /dev/null @@ -1,282 +0,0 @@ -use arrow::array::make_array; -use arrow::array::Array; -use arrow::array::ArrayRef; -use arrow::array::AsArray; -use arrow::array::Float32Array; -use arrow::array::Float64Array; -use arrow::array::Int16Array; -use arrow::array::Int32Array; -use arrow::array::Int64Array; -use arrow::array::Int8Array; -use arrow::array::ListArray; -use arrow::array::StringArray; -use arrow::array::StructArray; -use arrow::array::UInt16Array; -use arrow::array::UInt32Array; -use arrow::array::UInt64Array; -use arrow::array::UInt8Array; -use arrow::datatypes::DataType; -use serde::ser::SerializeSeq; -use serde::ser::SerializeStruct; - -use super::TypeInfo; - -#[derive(Debug, Clone, PartialEq)] -pub struct TypedValue<'a> { - pub value: &'a ArrayRef, - pub type_info: &'a TypeInfo, -} - -impl serde::Serialize for TypedValue<'_> { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match &self.type_info.data_type { - DataType::UInt8 => { - let array: &UInt8Array = self.value.as_primitive(); - debug_assert!(array.len() == 1, "array length was: {}", array.len()); - let number = array.value(0); - serializer.serialize_u8(number) - } - DataType::UInt16 => { - let array: &UInt16Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_u16(number) - } - DataType::UInt32 => { - let array: &UInt32Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_u32(number) - } - DataType::UInt64 => { - let array: &UInt64Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_u64(number) - } - DataType::Int8 => { - let array: &Int8Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_i8(number) - } - DataType::Int16 => { - let array: &Int16Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_i16(number) - } - DataType::Int32 => { - let array: &Int32Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_i32(number) - } - DataType::Int64 => { - let array: &Int64Array = self.value.as_primitive(); - debug_assert!(array.len() == 1, "array was: {:#?}", array); - let number = array.value(0); - serializer.serialize_i64(number) - } - DataType::Float32 => { - let array: &Float32Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_f32(number) - } - DataType::Float64 => { - let array: &Float64Array = self.value.as_primitive(); - debug_assert!(array.len() == 1); - let number = array.value(0); - serializer.serialize_f64(number) - } - DataType::Utf8 => { - let int_array: &StringArray = self.value.as_string(); - let string = int_array.value(0); - serializer.serialize_str(string) - } - DataType::List(_field) => { - let list_array: &ListArray = self.value.as_list(); - let mut s = serializer.serialize_seq(Some(list_array.len()))?; - for root in list_array.iter() { - if let Some(values) = root { - match values.data_type() { - DataType::UInt8 => { - let values: &UInt8Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::UInt16 => { - let values: &UInt16Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::UInt32 => { - let values: &UInt32Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::UInt64 => { - let values: &UInt64Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Int8 => { - let values: &Int8Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Int16 => { - let values: &Int16Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Int32 => { - let values: &Int32Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Int64 => { - let values: &Int64Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Float32 => { - let values: &Float32Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Float64 => { - let values: &Float64Array = values.as_primitive(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Utf8 => { - let values: &StringArray = values.as_string(); - for value in values.iter() { - if let Some(value) = value { - s.serialize_element(&value)?; - } else { - todo!("Implement null management"); - } - } - } - DataType::Struct(_fields) => { - let list_array: ListArray = self.type_info.defaults.clone().into(); - s.serialize_element(&TypedValue { - value: &values, - type_info: &TypeInfo { - data_type: values.data_type().clone(), - defaults: list_array.value(0).to_data(), - }, - })?; - } - op => todo!("Implement additional type: {:?}", op), - } - } else { - todo!("Implement null management"); - } - } - s.end() - } - DataType::Struct(fields) => { - /// Serde requires that struct and field names are known at - /// compile time with a `'static` lifetime, which is not - /// possible in this case. Thus, we need to use dummy names - /// instead. - /// - /// The actual names do not really matter because - /// the CDR format of ROS2 does not encode struct or field - /// names. - const DUMMY_STRUCT_NAME: &str = "struct"; - const DUMMY_FIELD_NAME: &str = "field"; - - let struct_array: &StructArray = self.value.as_struct(); - let mut s = serializer.serialize_struct(DUMMY_STRUCT_NAME, fields.len())?; - let defaults: StructArray = self.type_info.defaults.clone().into(); - for field in fields.iter() { - let default = match defaults.column_by_name(field.name()) { - Some(value) => value.to_data(), - None => { - return Err(serde::ser::Error::custom(format!( - "missing field {} for serialization", - &field.name() - ))) - } - }; - let value = make_array(default.clone()); - let field_value = match struct_array.column_by_name(field.name()) { - Some(value) => value, - None => &value, - }; - - s.serialize_field( - DUMMY_FIELD_NAME, - &TypedValue { - value: &field_value.clone(), - type_info: &TypeInfo { - data_type: field.data_type().clone(), - defaults: default, - }, - }, - )?; - } - s.end() - } - _ => todo!(), - } - } -} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs new file mode 100644 index 00000000..517d2e6a --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs @@ -0,0 +1,186 @@ +use std::marker::PhantomData; + +use arrow::{ + array::{Array, ArrayRef, AsArray, PrimitiveArray}, + datatypes::{self, ArrowPrimitiveType}, +}; +use dora_ros2_bridge_msg_gen::types::{ + primitives::{BasicType, NestableType}, + sequences, +}; +use serde::ser::SerializeTuple; + +use super::error; + +/// Serialize an array with known size as tuple. +pub struct ArraySerializeWrapper<'a> { + pub array_info: &'a sequences::Array, + pub column: &'a ArrayRef, +} + +impl serde::Serialize for ArraySerializeWrapper<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let entry = if let Some(list) = self.column.as_list_opt::() { + // should match the length of the outer struct + assert_eq!(list.len(), 1); + list.value(0) + } else { + // try as large list + let list = self + .column + .as_list_opt::() + .ok_or_else(|| error("value is not compatible with expected Array type"))?; + // should match the length of the outer struct + assert_eq!(list.len(), 1); + list.value(0) + }; + + match &self.array_info.value_type { + NestableType::BasicType(t) => match t { + BasicType::I8 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I16 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I32 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I64 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U8 | BasicType::Char | BasicType::Byte => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U16 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U32 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U64 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::F32 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::F64 => BasicArrayAsTuple { + len: self.array_info.size, + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::Bool => BoolArrayAsTuple { + len: self.array_info.size, + value: &entry, + } + .serialize(serializer), + }, + NestableType::NamedType(_) => todo!("serializing arrays of NestableType::NamedType"), + NestableType::NamespacedType(_) => { + todo!("serializing arrays of NestableType::NamespacedType") + } + NestableType::GenericString(_) => { + todo!("serializing arrays of NestableType::GenericString") + } + } + } +} + +/// Serializes a primitive array with known size as tuple. +struct BasicArrayAsTuple<'a, T> { + len: usize, + value: &'a ArrayRef, + ty: PhantomData, +} + +impl serde::Serialize for BasicArrayAsTuple<'_, T> +where + T: ArrowPrimitiveType, + T::Native: serde::Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_tuple(self.len)?; + let array: &PrimitiveArray = self + .value + .as_primitive_opt() + .ok_or_else(|| error("not a primitive array"))?; + if array.len() != self.len { + return Err(error(format!( + "expected array with length {}, got length {}", + self.len, + array.len() + ))); + } + + for value in array.values() { + seq.serialize_element(value)?; + } + + seq.end() + } +} + +struct BoolArrayAsTuple<'a> { + len: usize, + value: &'a ArrayRef, +} + +impl serde::Serialize for BoolArrayAsTuple<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_tuple(self.len)?; + let array = self + .value + .as_boolean_opt() + .ok_or_else(|| error("not a boolean array"))?; + if array.len() != self.len { + return Err(error(format!( + "expected array with length {}, got length {}", + self.len, + array.len() + ))); + } + + for value in array.values() { + seq.serialize_element(&value)?; + } + + seq.end() + } +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/defaults.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/defaults.rs new file mode 100644 index 00000000..2cc1e6b7 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/defaults.rs @@ -0,0 +1,237 @@ +use arrow::{ + array::{ + make_array, Array, ArrayData, BooleanArray, Float32Array, Float64Array, Int16Array, + Int32Array, Int64Array, Int8Array, ListArray, StringArray, StructArray, UInt16Array, + UInt32Array, UInt64Array, UInt8Array, + }, + buffer::{OffsetBuffer, ScalarBuffer}, + compute::concat, + datatypes::Field, +}; +use dora_ros2_bridge_msg_gen::types::{ + primitives::{BasicType, NestableType}, + MemberType, Message, +}; +use eyre::{Context, ContextCompat, Result}; +use std::{collections::HashMap, sync::Arc, vec}; + +pub fn default_for_member( + m: &dora_ros2_bridge_msg_gen::types::Member, + package_name: &str, + messages: &HashMap>, +) -> eyre::Result { + let value = match &m.r#type { + MemberType::NestableType(t) => match t { + NestableType::BasicType(_) | NestableType::GenericString(_) => match &m + .default + .as_deref() + { + Some([]) => eyre::bail!("empty default value not supported"), + Some([default]) => preset_default_for_basic_type(t, default) + .with_context(|| format!("failed to parse default value for `{}`", m.name))?, + Some(_) => eyre::bail!( + "there should be only a single default value for non-sequence types" + ), + None => default_for_nestable_type(t, package_name, messages, 1)?, + }, + NestableType::NamedType(_) => { + if m.default.is_some() { + eyre::bail!("default values for nested types are not supported") + } else { + default_for_nestable_type(t, package_name, messages, 1)? + } + } + NestableType::NamespacedType(_) => { + default_for_nestable_type(t, package_name, messages, 1)? + } + }, + MemberType::Array(array) => list_default_values( + m, + &array.value_type, + package_name, + messages, + Some(array.size), + )?, + MemberType::Sequence(seq) => { + list_default_values(m, &seq.value_type, package_name, messages, None)? + } + MemberType::BoundedSequence(seq) => list_default_values( + m, + &seq.value_type, + package_name, + messages, + Some(seq.max_size), + )?, + }; + Ok(value) +} + +fn default_for_nestable_type( + t: &NestableType, + package_name: &str, + messages: &HashMap>, + size: usize, +) -> Result { + let empty = HashMap::new(); + let package_messages = messages.get(package_name).unwrap_or(&empty); + let array = match t { + NestableType::BasicType(t) => match t { + BasicType::I8 => Int8Array::from(vec![0; size]).into(), + BasicType::I16 => Int16Array::from(vec![0; size]).into(), + BasicType::I32 => Int32Array::from(vec![0; size]).into(), + BasicType::I64 => Int64Array::from(vec![0; size]).into(), + BasicType::U8 => UInt8Array::from(vec![0; size]).into(), + BasicType::U16 => UInt16Array::from(vec![0; size]).into(), + BasicType::U32 => UInt32Array::from(vec![0; size]).into(), + BasicType::U64 => UInt64Array::from(vec![0; size]).into(), + BasicType::F32 => Float32Array::from(vec![0.; size]).into(), + BasicType::F64 => Float64Array::from(vec![0.; size]).into(), + BasicType::Char => StringArray::from(vec![""]).into(), + BasicType::Byte => UInt8Array::from(vec![0u8; size]).into(), + BasicType::Bool => BooleanArray::from(vec![false; size]).into(), + }, + NestableType::GenericString(_) => StringArray::from(vec![""]).into(), + NestableType::NamedType(name) => { + let referenced_message = package_messages + .get(&name.0) + .context("unknown referenced message")?; + + default_for_referenced_message(referenced_message, package_name, messages)? + } + NestableType::NamespacedType(t) => { + let referenced_package_messages = messages.get(&t.package).unwrap_or(&empty); + let referenced_message = referenced_package_messages + .get(&t.name) + .context("unknown referenced message")?; + default_for_referenced_message(referenced_message, &t.package, messages)? + } + }; + Ok(array) +} + +fn preset_default_for_basic_type(t: &NestableType, preset: &str) -> Result { + Ok(match t { + NestableType::BasicType(t) => match t { + BasicType::I8 => Int8Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::I16 => Int16Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::I32 => Int32Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::I64 => Int64Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::U8 => UInt8Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::U16 => UInt16Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::U32 => UInt32Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::U64 => UInt64Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::F32 => Float32Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::F64 => Float64Array::from(vec![preset + .parse::() + .context("Could not parse preset default value")?]) + .into(), + BasicType::Char => StringArray::from(vec![preset]).into(), + BasicType::Byte => UInt8Array::from(preset.as_bytes().to_owned()).into(), + BasicType::Bool => BooleanArray::from(vec![preset + .parse::() + .context("could not parse preset default value")?]) + .into(), + }, + NestableType::GenericString(_) => StringArray::from(vec![preset]).into(), + _ => todo!("preset_default_for_basic_type (other)"), + }) +} + +fn default_for_referenced_message( + referenced_message: &Message, + package_name: &str, + messages: &HashMap>, +) -> eyre::Result { + let fields: Vec<(Arc, Arc)> = referenced_message + .members + .iter() + .map(|m| { + let default = default_for_member(m, package_name, messages)?; + Result::<_, eyre::Report>::Ok(( + Arc::new(Field::new( + m.name.clone(), + default.data_type().clone(), + true, + )), + make_array(default), + )) + }) + .collect::>()?; + + let struct_array: StructArray = fields.into(); + Ok(struct_array.into()) +} + +fn list_default_values( + m: &dora_ros2_bridge_msg_gen::types::Member, + value_type: &NestableType, + package_name: &str, + messages: &HashMap>, + size: Option, +) -> Result { + let defaults = match &m.default.as_deref() { + Some([]) => eyre::bail!("empty default value not supported"), + Some(defaults) => { + let raw_array: Vec> = defaults + .iter() + .map(|default| { + preset_default_for_basic_type(value_type, default) + .with_context(|| format!("failed to parse default value for `{}`", m.name)) + .map(make_array) + }) + .collect::>()?; + let default_values = concat( + raw_array + .iter() + .map(|data| data.as_ref()) + .collect::>() + .as_slice(), + ) + .context("Failed to concatenate default list value")?; + default_values.to_data() + } + None => { + let size = size.unwrap_or(1); + let default_nested_type = + default_for_nestable_type(value_type, package_name, messages, size)?; + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, size as i32])); + + let field = Arc::new(Field::new( + "item", + default_nested_type.data_type().clone(), + true, + )); + let list = ListArray::new(field, offsets, make_array(default_nested_type), None); + list.to_data() + } + }; + + Ok(defaults) +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs new file mode 100644 index 00000000..ab7556d4 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -0,0 +1,181 @@ +use std::{borrow::Cow, collections::HashMap, fmt::Display}; + +use arrow::array::{Array, ArrayRef, AsArray}; +use dora_ros2_bridge_msg_gen::types::{ + primitives::{GenericString, NestableType}, + MemberType, +}; +use serde::ser::SerializeStruct; + +use super::{TypeInfo, DUMMY_FIELD_NAME, DUMMY_STRUCT_NAME}; + +mod array; +mod defaults; +mod primitive; +mod sequence; + +#[derive(Debug, Clone)] +pub struct TypedValue<'a> { + pub value: &'a ArrayRef, + pub type_info: &'a TypeInfo<'a>, +} + +impl serde::Serialize for TypedValue<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let empty = HashMap::new(); + let package_messages = self + .type_info + .messages + .get(self.type_info.package_name.as_ref()) + .unwrap_or(&empty); + let message = package_messages + .get(self.type_info.message_name.as_ref()) + .ok_or_else(|| { + error(format!( + "could not find message type {}::{}", + self.type_info.package_name, self.type_info.message_name + )) + })?; + + let input = self + .value + .as_struct_opt() + .ok_or_else(|| error("expected struct array"))?; + for column_name in input.column_names() { + if !message.members.iter().any(|m| m.name == column_name) { + return Err(error(format!( + "given struct has unknown field {column_name}" + )))?; + } + } + if input.is_empty() { + // TODO: publish default value + return Err(error("given struct is empty"))?; + } + if input.len() > 1 { + return Err(error(format!( + "expected single struct instance, got struct array with {} entries", + input.len() + )))?; + } + let mut s = serializer.serialize_struct(DUMMY_STRUCT_NAME, message.members.len())?; + for field in message.members.iter() { + let column: Cow<_> = match input.column_by_name(&field.name) { + Some(input) => Cow::Borrowed(input), + None => { + let default = defaults::default_for_member( + field, + &self.type_info.package_name, + &self.type_info.messages, + ) + .map_err(error)?; + Cow::Owned(arrow::array::make_array(default)) + } + }; + + match &field.r#type { + MemberType::NestableType(t) => match t { + NestableType::BasicType(t) => { + s.serialize_field( + DUMMY_FIELD_NAME, + &primitive::SerializeWrapper { + t, + column: column.as_ref(), + }, + )?; + } + NestableType::NamedType(name) => { + let referenced_value = &TypedValue { + value: column.as_ref(), + type_info: &TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }, + }; + s.serialize_field(DUMMY_FIELD_NAME, &referenced_value)?; + } + NestableType::NamespacedType(reference) => { + if reference.namespace != "msg" { + return Err(error(format!( + "struct field {} references non-message type {reference:?}", + field.name + ))); + } + + let referenced_value: &TypedValue<'_> = &TypedValue { + value: column.as_ref(), + type_info: &TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }, + }; + s.serialize_field(DUMMY_FIELD_NAME, &referenced_value)?; + } + NestableType::GenericString(t) => match t { + GenericString::String | GenericString::BoundedString(_) => { + let string = if let Some(string_array) = column.as_string_opt::() { + // should match the length of the outer struct array + assert_eq!(string_array.len(), 1); + string_array.value(0) + } else { + // try again with large offset type + let string_array = column + .as_string_opt::() + .ok_or_else(|| error("expected string array"))?; + // should match the length of the outer struct array + assert_eq!(string_array.len(), 1); + string_array.value(0) + }; + s.serialize_field(DUMMY_FIELD_NAME, string)?; + } + GenericString::WString => todo!("serializing WString types"), + GenericString::BoundedWString(_) => { + todo!("serializing BoundedWString types") + } + }, + }, + dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { + s.serialize_field( + DUMMY_FIELD_NAME, + &array::ArraySerializeWrapper { + array_info: a, + column: column.as_ref(), + }, + )?; + } + dora_ros2_bridge_msg_gen::types::MemberType::Sequence(v) => { + s.serialize_field( + DUMMY_FIELD_NAME, + &sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + }, + )?; + } + dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(v) => { + s.serialize_field( + DUMMY_FIELD_NAME, + &sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + }, + )?; + } + } + } + s.end() + } +} + +fn error(e: T) -> E +where + T: Display, + E: serde::ser::Error, +{ + serde::ser::Error::custom(e) +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/primitive.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/primitive.rs new file mode 100644 index 00000000..a13bf444 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/primitive.rs @@ -0,0 +1,79 @@ +use arrow::{ + array::{ArrayRef, AsArray}, + datatypes::{self, ArrowPrimitiveType}, +}; +use dora_ros2_bridge_msg_gen::types::primitives::BasicType; + +pub struct SerializeWrapper<'a> { + pub t: &'a BasicType, + pub column: &'a ArrayRef, +} + +impl serde::Serialize for SerializeWrapper<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self.t { + BasicType::I8 => { + serializer.serialize_i8(as_single_primitive::(self.column)?) + } + BasicType::I16 => serializer + .serialize_i16(as_single_primitive::(self.column)?), + BasicType::I32 => serializer + .serialize_i32(as_single_primitive::(self.column)?), + BasicType::I64 => serializer + .serialize_i64(as_single_primitive::(self.column)?), + BasicType::U8 | BasicType::Char | BasicType::Byte => serializer + .serialize_u8(as_single_primitive::(self.column)?), + BasicType::U16 => serializer + .serialize_u16(as_single_primitive::( + self.column, + )?), + BasicType::U32 => serializer + .serialize_u32(as_single_primitive::( + self.column, + )?), + BasicType::U64 => serializer + .serialize_u64(as_single_primitive::( + self.column, + )?), + BasicType::F32 => serializer + .serialize_f32(as_single_primitive::( + self.column, + )?), + BasicType::F64 => serializer + .serialize_f64(as_single_primitive::( + self.column, + )?), + BasicType::Bool => { + let array = self.column.as_boolean_opt().ok_or_else(|| { + serde::ser::Error::custom( + "value is not compatible with expected `BooleanArray` type", + ) + })?; + // should match the length of the outer struct + assert_eq!(array.len(), 1); + let field_value = array.value(0); + serializer.serialize_bool(field_value) + } + } + } +} + +fn as_single_primitive(column: &ArrayRef) -> Result +where + T: ArrowPrimitiveType, + E: serde::ser::Error, +{ + let array: &arrow::array::PrimitiveArray = column.as_primitive_opt().ok_or_else(|| { + serde::ser::Error::custom(format!( + "value is not compatible with expected `{}` type", + std::any::type_name::() + )) + })?; + // should match the length of the outer struct + assert_eq!(array.len(), 1); + let number = array.value(0); + Ok(number) +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs new file mode 100644 index 00000000..fd8dd591 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs @@ -0,0 +1,152 @@ +use std::marker::PhantomData; + +use arrow::{ + array::{Array, ArrayRef, AsArray, PrimitiveArray}, + datatypes::{self, ArrowPrimitiveType}, +}; +use dora_ros2_bridge_msg_gen::types::primitives::{BasicType, NestableType}; +use serde::ser::{SerializeSeq, SerializeTuple}; + +use super::error; + +/// Serialize a variable-sized sequence. +pub struct SequenceSerializeWrapper<'a> { + pub item_type: &'a NestableType, + pub column: &'a ArrayRef, +} + +impl serde::Serialize for SequenceSerializeWrapper<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let entry = if let Some(list) = self.column.as_list_opt::() { + // should match the length of the outer struct + assert_eq!(list.len(), 1); + list.value(0) + } else { + // try as large list + let list = self + .column + .as_list_opt::() + .ok_or_else(|| error("value is not compatible with expected Array type"))?; + // should match the length of the outer struct + assert_eq!(list.len(), 1); + list.value(0) + }; + match &self.item_type { + NestableType::BasicType(t) => match t { + BasicType::I8 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I16 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I32 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::I64 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U8 | BasicType::Char | BasicType::Byte => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U16 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U32 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::U64 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::F32 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::F64 => BasicSequence { + value: &entry, + ty: PhantomData::, + } + .serialize(serializer), + BasicType::Bool => BoolArray { value: &entry }.serialize(serializer), + }, + NestableType::NamedType(_) => todo!("serializing NestableType::NamedType sequences"), + NestableType::NamespacedType(_) => { + todo!("serializing NestableType::NamespacedType sequences") + } + NestableType::GenericString(_) => { + todo!("serializing NestableType::Genericstring sequences") + } + } + } +} + +struct BasicSequence<'a, T> { + value: &'a ArrayRef, + ty: PhantomData, +} + +impl serde::Serialize for BasicSequence<'_, T> +where + T: ArrowPrimitiveType, + T::Native: serde::Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let array: &PrimitiveArray = self + .value + .as_primitive_opt() + .ok_or_else(|| error("not a primitive array"))?; + + let mut seq = serializer.serialize_seq(Some(array.len()))?; + + for value in array.values() { + seq.serialize_element(value)?; + } + + seq.end() + } +} + +struct BoolArray<'a> { + value: &'a ArrayRef, +} + +impl serde::Serialize for BoolArray<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let array = self + .value + .as_boolean_opt() + .ok_or_else(|| error("not a boolean array"))?; + let mut seq = serializer.serialize_tuple(array.len())?; + + for value in array.values() { + seq.serialize_element(&value)?; + } + + seq.end() + } +} From 4b8dbfc7c26e1fbe8a8186deabd2a08c1cb4e605 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Sun, 21 Jan 2024 16:24:52 +0100 Subject: [PATCH 082/104] Add support for (de)serializing arrays/sequences of structs and strings --- .../python/src/typed/deserialize/array.rs | 15 ++- .../python/src/typed/deserialize/mod.rs | 6 +- .../python/src/typed/deserialize/sequence.rs | 66 ++++++++++++-- .../python/src/typed/serialize/array.rs | 91 +++++++++++++++++-- .../python/src/typed/serialize/mod.rs | 3 + .../python/src/typed/serialize/sequence.rs | 88 ++++++++++++++++-- 6 files changed, 239 insertions(+), 30 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs index 79030abe..170092dc 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/array.rs @@ -1,9 +1,14 @@ use arrow::array::ArrayData; use dora_ros2_bridge_msg_gen::types::sequences; +use crate::typed::TypeInfo; + use super::sequence::SequenceVisitor; -pub struct ArrayDeserializer<'a>(pub &'a sequences::Array); +pub struct ArrayDeserializer<'a> { + pub array_type: &'a sequences::Array, + pub type_info: &'a TypeInfo<'a>, +} impl<'de> serde::de::DeserializeSeed<'de> for ArrayDeserializer<'_> { type Value = ArrayData; @@ -12,6 +17,12 @@ impl<'de> serde::de::DeserializeSeed<'de> for ArrayDeserializer<'_> { where D: serde::Deserializer<'de>, { - deserializer.deserialize_tuple(self.0.size, SequenceVisitor(&self.0.value_type)) + deserializer.deserialize_tuple( + self.array_type.size, + SequenceVisitor { + item_type: &self.array_type.value_type, + type_info: self.type_info, + }, + ) } } diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs index 93a94eee..ca74e8e5 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs @@ -125,13 +125,13 @@ impl<'a, 'de> serde::de::Visitor<'de> for StructVisitor<'a> { } }, dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { - data.next_element_seed(array::ArrayDeserializer(a))? + data.next_element_seed(array::ArrayDeserializer{ array_type : a, type_info: self.type_info})? }, dora_ros2_bridge_msg_gen::types::MemberType::Sequence(s) => { - data.next_element_seed(sequence::SequenceDeserializer(&s.value_type))? + data.next_element_seed(sequence::SequenceDeserializer{item_type: &s.value_type, type_info: self.type_info})? }, dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(s) => { - data.next_element_seed(sequence::SequenceDeserializer(&s.value_type))? + data.next_element_seed(sequence::SequenceDeserializer{ item_type: &s.value_type, type_info: self.type_info})? }, }; diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs index 47395012..ee918ef3 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs @@ -5,8 +5,16 @@ use arrow::{ use core::fmt; use dora_ros2_bridge_msg_gen::types::primitives::{self, BasicType, NestableType}; use serde::Deserialize; +use std::{borrow::Cow, ops::Deref}; -pub struct SequenceDeserializer<'a>(pub &'a NestableType); +use crate::typed::TypeInfo; + +use super::{error, StructDeserializer}; + +pub struct SequenceDeserializer<'a> { + pub item_type: &'a NestableType, + pub type_info: &'a TypeInfo<'a>, +} impl<'de> serde::de::DeserializeSeed<'de> for SequenceDeserializer<'_> { type Value = ArrayData; @@ -15,11 +23,17 @@ impl<'de> serde::de::DeserializeSeed<'de> for SequenceDeserializer<'_> { where D: serde::Deserializer<'de>, { - deserializer.deserialize_seq(SequenceVisitor(self.0)) + deserializer.deserialize_seq(SequenceVisitor { + item_type: self.item_type, + type_info: self.type_info, + }) } } -pub struct SequenceVisitor<'a>(pub &'a NestableType); +pub struct SequenceVisitor<'a> { + pub item_type: &'a NestableType, + pub type_info: &'a TypeInfo<'a>, +} impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { type Value = ArrayData; @@ -32,7 +46,7 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { where A: serde::de::SeqAccess<'de>, { - match &self.0 { + match &self.item_type { NestableType::BasicType(t) => match t { BasicType::I8 => deserialize_primitive_seq::<_, datatypes::Int8Type>(seq), BasicType::I16 => deserialize_primitive_seq::<_, datatypes::Int16Type>(seq), @@ -54,9 +68,30 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { Ok(array.finish().into()) } }, - NestableType::NamedType(_) => todo!("deserialize sequence of NestableType::NamedType"), - NestableType::NamespacedType(_) => { - todo!("deserialize sequence of NestableType::NamedspacedType") + NestableType::NamedType(name) => { + let deserializer = StructDeserializer { + type_info: Cow::Owned(TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }), + }; + deserialize_struct_array(&mut seq, deserializer) + } + NestableType::NamespacedType(reference) => { + if reference.namespace != "msg" { + return Err(error(format!( + "sequence item references non-message type {reference:?}", + ))); + } + let deserializer = StructDeserializer { + type_info: Cow::Owned(TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }), + }; + deserialize_struct_array(&mut seq, deserializer) } NestableType::GenericString(t) => match t { primitives::GenericString::String | primitives::GenericString::BoundedString(_) => { @@ -75,6 +110,23 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { } } +fn deserialize_struct_array<'de, A>( + seq: &mut A, + deserializer: StructDeserializer<'_>, +) -> Result>::Error> +where + A: serde::de::SeqAccess<'de>, +{ + let mut values = Vec::new(); + while let Some(value) = seq.next_element_seed(deserializer.clone())? { + values.push(arrow::array::make_array(value)); + } + let refs: Vec<_> = values.iter().map(|a| a.deref()).collect(); + arrow::compute::concat(&refs) + .map(|a| a.to_data()) + .map_err(super::error) +} + fn deserialize_primitive_seq<'de, S, T>( mut seq: S, ) -> Result>::Error> diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs index 517d2e6a..d1454fd8 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs @@ -1,21 +1,24 @@ -use std::marker::PhantomData; +use std::{borrow::Cow, marker::PhantomData, sync::Arc}; use arrow::{ - array::{Array, ArrayRef, AsArray, PrimitiveArray}, + array::{Array, ArrayRef, AsArray, OffsetSizeTrait, PrimitiveArray}, datatypes::{self, ArrowPrimitiveType}, }; use dora_ros2_bridge_msg_gen::types::{ - primitives::{BasicType, NestableType}, + primitives::{BasicType, GenericString, NestableType}, sequences, }; use serde::ser::SerializeTuple; -use super::error; +use crate::typed::TypeInfo; + +use super::{error, TypedValue}; /// Serialize an array with known size as tuple. pub struct ArraySerializeWrapper<'a> { pub array_info: &'a sequences::Array, pub column: &'a ArrayRef, + pub type_info: &'a TypeInfo<'a>, } impl serde::Serialize for ArraySerializeWrapper<'_> { @@ -106,13 +109,67 @@ impl serde::Serialize for ArraySerializeWrapper<'_> { } .serialize(serializer), }, - NestableType::NamedType(_) => todo!("serializing arrays of NestableType::NamedType"), - NestableType::NamespacedType(_) => { - todo!("serializing arrays of NestableType::NamespacedType") + NestableType::NamedType(name) => { + let array = entry + .as_struct_opt() + .ok_or_else(|| error("not a struct array"))?; + let mut seq = serializer.serialize_tuple(self.array_info.size)?; + for i in 0..array.len() { + let row = array.slice(i, 1); + seq.serialize_element(&TypedValue { + value: &(Arc::new(row) as ArrayRef), + type_info: &crate::typed::TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }, + })?; + } + seq.end() } - NestableType::GenericString(_) => { - todo!("serializing arrays of NestableType::GenericString") + NestableType::NamespacedType(reference) => { + if reference.namespace != "msg" { + return Err(error(format!( + "sequence references non-message type {reference:?}" + ))); + } + + let array = entry + .as_struct_opt() + .ok_or_else(|| error("not a struct array"))?; + let mut seq = serializer.serialize_tuple(self.array_info.size)?; + for i in 0..array.len() { + let row = array.slice(i, 1); + seq.serialize_element(&TypedValue { + value: &(Arc::new(row) as ArrayRef), + type_info: &crate::typed::TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }, + })?; + } + seq.end() } + NestableType::GenericString(s) => match s { + GenericString::String | GenericString::BoundedString(_) => { + match entry.as_string_opt::() { + Some(array) => { + serialize_arrow_string(serializer, array, self.array_info.size) + } + None => { + let array = entry + .as_string_opt::() + .ok_or_else(|| error("expected string array"))?; + serialize_arrow_string(serializer, array, self.array_info.size) + } + } + } + GenericString::WString => { + todo!("serializing WString sequences") + } + GenericString::BoundedWString(_) => todo!("serializing BoundedWString sequences"), + }, } } } @@ -184,3 +241,19 @@ impl serde::Serialize for BoolArrayAsTuple<'_> { seq.end() } } + +fn serialize_arrow_string( + serializer: S, + array: &arrow::array::GenericByteArray>, + array_len: usize, +) -> Result<::Ok, ::Error> +where + S: serde::Serializer, + O: OffsetSizeTrait, +{ + let mut seq = serializer.serialize_tuple(array_len)?; + for s in array.iter() { + seq.serialize_element(s.unwrap_or_default())?; + } + seq.end() +} diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs index ab7556d4..0382b355 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -145,6 +145,7 @@ impl serde::Serialize for TypedValue<'_> { &array::ArraySerializeWrapper { array_info: a, column: column.as_ref(), + type_info: self.type_info, }, )?; } @@ -154,6 +155,7 @@ impl serde::Serialize for TypedValue<'_> { &sequence::SequenceSerializeWrapper { item_type: &v.value_type, column: column.as_ref(), + type_info: self.type_info, }, )?; } @@ -163,6 +165,7 @@ impl serde::Serialize for TypedValue<'_> { &sequence::SequenceSerializeWrapper { item_type: &v.value_type, column: column.as_ref(), + type_info: self.type_info, }, )?; } diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs index fd8dd591..3fb6bdae 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs @@ -1,18 +1,21 @@ -use std::marker::PhantomData; +use std::{borrow::Cow, marker::PhantomData, sync::Arc}; use arrow::{ - array::{Array, ArrayRef, AsArray, PrimitiveArray}, + array::{Array, ArrayRef, AsArray, OffsetSizeTrait, PrimitiveArray}, datatypes::{self, ArrowPrimitiveType}, }; -use dora_ros2_bridge_msg_gen::types::primitives::{BasicType, NestableType}; +use dora_ros2_bridge_msg_gen::types::primitives::{BasicType, GenericString, NestableType}; use serde::ser::{SerializeSeq, SerializeTuple}; -use super::error; +use crate::typed::TypeInfo; + +use super::{error, TypedValue}; /// Serialize a variable-sized sequence. pub struct SequenceSerializeWrapper<'a> { pub item_type: &'a NestableType, pub column: &'a ArrayRef, + pub type_info: &'a TypeInfo<'a>, } impl serde::Serialize for SequenceSerializeWrapper<'_> { @@ -88,17 +91,84 @@ impl serde::Serialize for SequenceSerializeWrapper<'_> { .serialize(serializer), BasicType::Bool => BoolArray { value: &entry }.serialize(serializer), }, - NestableType::NamedType(_) => todo!("serializing NestableType::NamedType sequences"), - NestableType::NamespacedType(_) => { - todo!("serializing NestableType::NamespacedType sequences") + NestableType::NamedType(name) => { + let array = entry + .as_struct_opt() + .ok_or_else(|| error("not a struct array"))?; + let mut seq = serializer.serialize_seq(Some(array.len()))?; + for i in 0..array.len() { + let row = array.slice(i, 1); + seq.serialize_element(&TypedValue { + value: &(Arc::new(row) as ArrayRef), + type_info: &crate::typed::TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }, + })?; + } + seq.end() } - NestableType::GenericString(_) => { - todo!("serializing NestableType::Genericstring sequences") + NestableType::NamespacedType(reference) => { + if reference.namespace != "msg" { + return Err(error(format!( + "sequence references non-message type {reference:?}" + ))); + } + + let array = entry + .as_struct_opt() + .ok_or_else(|| error("not a struct array"))?; + let mut seq = serializer.serialize_seq(Some(array.len()))?; + for i in 0..array.len() { + let row = array.slice(i, 1); + seq.serialize_element(&TypedValue { + value: &(Arc::new(row) as ArrayRef), + type_info: &crate::typed::TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }, + })?; + } + seq.end() } + NestableType::GenericString(s) => match s { + GenericString::String | GenericString::BoundedString(_) => { + match entry.as_string_opt::() { + Some(array) => serialize_arrow_string(serializer, array), + None => { + let array = entry + .as_string_opt::() + .ok_or_else(|| error("expected string array"))?; + serialize_arrow_string(serializer, array) + } + } + } + GenericString::WString => { + todo!("serializing WString sequences") + } + GenericString::BoundedWString(_) => todo!("serializing BoundedWString sequences"), + }, } } } +fn serialize_arrow_string( + serializer: S, + array: &arrow::array::GenericByteArray>, +) -> Result<::Ok, ::Error> +where + S: serde::Serializer, + O: OffsetSizeTrait, +{ + let mut seq = serializer.serialize_seq(Some(array.len()))?; + for s in array.iter() { + seq.serialize_element(s.unwrap_or_default())?; + } + seq.end() +} + struct BasicSequence<'a, T> { value: &'a ArrayRef, ty: PhantomData, From abc057dda2ebeda8a54c4cb947dbc9eb8cd502e1 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Sun, 21 Jan 2024 16:25:29 +0100 Subject: [PATCH 083/104] Remove commented out code --- .../python/src/typed/deserialize/mod.rs | 128 ------------------ 1 file changed, 128 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs index ca74e8e5..d8b9123d 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs @@ -155,134 +155,6 @@ impl<'a, 'de> serde::de::Visitor<'de> for StructVisitor<'a> { } } -// struct ListVisitor { -// field: Arc, -// defaults: ArrayData, -// } - -// impl<'de> serde::de::Visitor<'de> for ListVisitor { -// type Value = ArrayData; - -// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { -// formatter.write_str("an array encoded as sequence") -// } - -// fn visit_seq(self, mut data: A) -> Result -// where -// A: serde::de::SeqAccess<'de>, -// { -// let data = match self.field.data_type().clone() { -// DataType::UInt8 => { -// let mut array = UInt8Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::UInt16 => { -// let mut array = UInt16Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::UInt32 => { -// let mut array = UInt32Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::UInt64 => { -// let mut array = UInt64Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Int8 => { -// let mut array = Int8Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Int16 => { -// let mut array = Int16Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Int32 => { -// let mut array = Int32Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Int64 => { -// let mut array = Int64Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Float32 => { -// let mut array = Float32Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Float64 => { -// let mut array = Float64Builder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// DataType::Utf8 => { -// let mut array = StringBuilder::new(); -// while let Some(value) = data.next_element::()? { -// array.append_value(value); -// } -// Ok(array.finish().into()) -// } -// _ => { -// let mut buffer = vec![]; -// while let Some(value) = data.next_element_seed(StructDeserializer { -// type_info: TypeInfo { -// data_type: self.field.data_type().clone(), -// defaults: self.defaults.clone(), -// }, -// })? { -// let element = make_array(value); -// buffer.push(element); -// } - -// concat( -// buffer -// .iter() -// .map(|data| data.as_ref()) -// .collect::>() -// .as_slice(), -// ) -// .map(|op| op.to_data()) -// } -// }; - -// if let Ok(values) = data { -// let offsets = OffsetBuffer::new(vec![0, values.len() as i32].into()); - -// let array = ListArray::new(self.field, offsets, make_array(values), None).to_data(); -// Ok(array) -// } else { -// Ok(self.defaults) // TODO: Better handle deserialization error -// } -// } -// } - fn error(e: T) -> E where T: Display, From e5a037afeabb9bd9882fcab07246aaa0f94e2776 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Sun, 21 Jan 2024 16:57:16 +0100 Subject: [PATCH 084/104] Improve error messages --- libraries/arrow-convert/src/from_impls.rs | 21 ++++++++++--------- .../python/src/typed/serialize/array.rs | 6 +++--- .../python/src/typed/serialize/sequence.rs | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/libraries/arrow-convert/src/from_impls.rs b/libraries/arrow-convert/src/from_impls.rs index db484c6b..01e8a951 100644 --- a/libraries/arrow-convert/src/from_impls.rs +++ b/libraries/arrow-convert/src/from_impls.rs @@ -39,7 +39,7 @@ impl TryFrom<&ArrowData> for u8 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive UInt8Type array")?; extract_single_primitive(array) } } @@ -48,7 +48,7 @@ impl TryFrom<&ArrowData> for u16 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive UInt16Type array")?; extract_single_primitive(array) } } @@ -57,7 +57,7 @@ impl TryFrom<&ArrowData> for u32 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive UInt32Type array")?; extract_single_primitive(array) } } @@ -66,7 +66,7 @@ impl TryFrom<&ArrowData> for u64 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive UInt64Type array")?; extract_single_primitive(array) } } @@ -75,7 +75,7 @@ impl TryFrom<&ArrowData> for i8 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive Int8Type array")?; extract_single_primitive(array) } } @@ -84,7 +84,7 @@ impl TryFrom<&ArrowData> for i16 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive Int16Type array")?; extract_single_primitive(array) } } @@ -93,7 +93,7 @@ impl TryFrom<&ArrowData> for i32 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive Int32Type array")?; extract_single_primitive(array) } } @@ -102,7 +102,7 @@ impl TryFrom<&ArrowData> for i64 { fn try_from(value: &ArrowData) -> Result { let array = value .as_primitive_opt::() - .context("not a primitive array")?; + .context("not a primitive Int64Type array")?; extract_single_primitive(array) } } @@ -127,8 +127,9 @@ impl<'a> TryFrom<&'a ArrowData> for &'a str { impl<'a> TryFrom<&'a ArrowData> for &'a [u8] { type Error = eyre::Report; fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = - value.as_primitive_opt().wrap_err("not a primitive array")?; + let array: &PrimitiveArray = value + .as_primitive_opt() + .wrap_err("not a primitive UInt8Type array")?; if array.null_count() != 0 { eyre::bail!("array has nulls"); } diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs index d1454fd8..3b2bf889 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/array.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, marker::PhantomData, sync::Arc}; +use std::{any::type_name, borrow::Cow, marker::PhantomData, sync::Arc}; use arrow::{ array::{Array, ArrayRef, AsArray, OffsetSizeTrait, PrimitiveArray}, @@ -35,7 +35,7 @@ impl serde::Serialize for ArraySerializeWrapper<'_> { let list = self .column .as_list_opt::() - .ok_or_else(|| error("value is not compatible with expected Array type"))?; + .ok_or_else(|| error("value is not compatible with expected array type"))?; // should match the length of the outer struct assert_eq!(list.len(), 1); list.value(0) @@ -194,7 +194,7 @@ where let array: &PrimitiveArray = self .value .as_primitive_opt() - .ok_or_else(|| error("not a primitive array"))?; + .ok_or_else(|| error(format!("not a primitive {} array", type_name::())))?; if array.len() != self.len { return Err(error(format!( "expected array with length {}, got length {}", diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs index 3fb6bdae..e80c7cf9 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs @@ -186,7 +186,7 @@ where let array: &PrimitiveArray = self .value .as_primitive_opt() - .ok_or_else(|| error("not a primitive array"))?; + .ok_or_else(|| error(format!("not a primitive {} array", type_name::())))?; let mut seq = serializer.serialize_seq(Some(array.len()))?; From 63188965c58bcd3ab178cca1cf09c7e3e2cd2c54 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Sun, 21 Jan 2024 17:03:06 +0100 Subject: [PATCH 085/104] Add support for serializing arrow `BinaryArray` types Arrow has a special `BinaryArray` type for storing lists of variable-sized binary data efficiently. This type is equivalent to a `ListArray` of `PrimitiveArray`, but it is a different data type. This commit updates the Python ROS2 serialization code to permit BinaryArray for all uint8 array or sequence fields. --- .../python/src/typed/serialize/sequence.rs | 70 +++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs index e80c7cf9..d42d45fb 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/sequence.rs @@ -1,8 +1,8 @@ -use std::{borrow::Cow, marker::PhantomData, sync::Arc}; +use std::{any::type_name, borrow::Cow, marker::PhantomData, sync::Arc}; use arrow::{ array::{Array, ArrayRef, AsArray, OffsetSizeTrait, PrimitiveArray}, - datatypes::{self, ArrowPrimitiveType}, + datatypes::{self, ArrowPrimitiveType, UInt8Type}, }; use dora_ros2_bridge_msg_gen::types::primitives::{BasicType, GenericString, NestableType}; use serde::ser::{SerializeSeq, SerializeTuple}; @@ -27,15 +27,23 @@ impl serde::Serialize for SequenceSerializeWrapper<'_> { // should match the length of the outer struct assert_eq!(list.len(), 1); list.value(0) - } else { - // try as large list - let list = self - .column - .as_list_opt::() - .ok_or_else(|| error("value is not compatible with expected Array type"))?; + } else if let Some(list) = self.column.as_list_opt::() { // should match the length of the outer struct assert_eq!(list.len(), 1); list.value(0) + } else if let Some(list) = self.column.as_binary_opt::() { + // should match the length of the outer struct + assert_eq!(list.len(), 1); + Arc::new(list.slice(0, 1)) as ArrayRef + } else if let Some(list) = self.column.as_binary_opt::() { + // should match the length of the outer struct + assert_eq!(list.len(), 1); + Arc::new(list.slice(0, 1)) as ArrayRef + } else { + return Err(error(format!( + "value is not compatible with expected sequence type: {:?}", + self.column + ))); }; match &self.item_type { NestableType::BasicType(t) => match t { @@ -59,11 +67,9 @@ impl serde::Serialize for SequenceSerializeWrapper<'_> { ty: PhantomData::, } .serialize(serializer), - BasicType::U8 | BasicType::Char | BasicType::Byte => BasicSequence { - value: &entry, - ty: PhantomData::, + BasicType::U8 | BasicType::Char | BasicType::Byte => { + ByteSequence { value: &entry }.serialize(serializer) } - .serialize(serializer), BasicType::U16 => BasicSequence { value: &entry, ty: PhantomData::, @@ -198,6 +204,46 @@ where } } +struct ByteSequence<'a> { + value: &'a ArrayRef, +} + +impl serde::Serialize for ByteSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if let Some(binary) = self.value.as_binary_opt::() { + serialize_binary(serializer, binary) + } else if let Some(binary) = self.value.as_binary_opt::() { + serialize_binary(serializer, binary) + } else { + BasicSequence { + value: self.value, + ty: PhantomData::, + } + .serialize(serializer) + } + } +} + +fn serialize_binary( + serializer: S, + binary: &arrow::array::GenericByteArray>, +) -> Result<::Ok, ::Error> +where + S: serde::Serializer, + O: OffsetSizeTrait, +{ + let mut seq = serializer.serialize_seq(Some(binary.len()))?; + + for value in binary.iter() { + seq.serialize_element(value.unwrap_or_default())?; + } + + seq.end() +} + struct BoolArray<'a> { value: &'a ArrayRef, } From 945b3f887b04475bed42a3feef45534833880fcc Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 23 Jan 2024 16:28:59 +0100 Subject: [PATCH 086/104] Changing `Struct` to `TupleStruct` --- .../python/src/typed/deserialize/mod.rs | 2 +- .../python/src/typed/serialize/mod.rs | 60 ++++++++----------- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs index d8b9123d..13288aa5 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs @@ -59,7 +59,7 @@ impl<'a, 'de> serde::de::Visitor<'de> for StructVisitor<'a> { type Value = ArrayData; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a struct encoded as sequence") + formatter.write_str("a struct encoded as TupleStruct") } fn visit_seq(self, mut data: A) -> Result diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs index 0382b355..7f38c36c 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -5,7 +5,7 @@ use dora_ros2_bridge_msg_gen::types::{ primitives::{GenericString, NestableType}, MemberType, }; -use serde::ser::SerializeStruct; +use serde::ser::{SerializeStruct, SerializeTupleStruct}; use super::{TypeInfo, DUMMY_FIELD_NAME, DUMMY_STRUCT_NAME}; @@ -61,7 +61,7 @@ impl serde::Serialize for TypedValue<'_> { input.len() )))?; } - let mut s = serializer.serialize_struct(DUMMY_STRUCT_NAME, message.members.len())?; + let mut s = serializer.serialize_tuple_struct(DUMMY_STRUCT_NAME, message.members.len())?; for field in message.members.iter() { let column: Cow<_> = match input.column_by_name(&field.name) { Some(input) => Cow::Borrowed(input), @@ -79,13 +79,10 @@ impl serde::Serialize for TypedValue<'_> { match &field.r#type { MemberType::NestableType(t) => match t { NestableType::BasicType(t) => { - s.serialize_field( - DUMMY_FIELD_NAME, - &primitive::SerializeWrapper { - t, - column: column.as_ref(), - }, - )?; + s.serialize_field(&primitive::SerializeWrapper { + t, + column: column.as_ref(), + })?; } NestableType::NamedType(name) => { let referenced_value = &TypedValue { @@ -96,7 +93,7 @@ impl serde::Serialize for TypedValue<'_> { messages: self.type_info.messages.clone(), }, }; - s.serialize_field(DUMMY_FIELD_NAME, &referenced_value)?; + s.serialize_field(&referenced_value)?; } NestableType::NamespacedType(reference) => { if reference.namespace != "msg" { @@ -114,7 +111,7 @@ impl serde::Serialize for TypedValue<'_> { messages: self.type_info.messages.clone(), }, }; - s.serialize_field(DUMMY_FIELD_NAME, &referenced_value)?; + s.serialize_field(&referenced_value)?; } NestableType::GenericString(t) => match t { GenericString::String | GenericString::BoundedString(_) => { @@ -131,7 +128,7 @@ impl serde::Serialize for TypedValue<'_> { assert_eq!(string_array.len(), 1); string_array.value(0) }; - s.serialize_field(DUMMY_FIELD_NAME, string)?; + s.serialize_field(string)?; } GenericString::WString => todo!("serializing WString types"), GenericString::BoundedWString(_) => { @@ -140,34 +137,25 @@ impl serde::Serialize for TypedValue<'_> { }, }, dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { - s.serialize_field( - DUMMY_FIELD_NAME, - &array::ArraySerializeWrapper { - array_info: a, - column: column.as_ref(), - type_info: self.type_info, - }, - )?; + s.serialize_field(&array::ArraySerializeWrapper { + array_info: a, + column: column.as_ref(), + type_info: self.type_info, + })?; } dora_ros2_bridge_msg_gen::types::MemberType::Sequence(v) => { - s.serialize_field( - DUMMY_FIELD_NAME, - &sequence::SequenceSerializeWrapper { - item_type: &v.value_type, - column: column.as_ref(), - type_info: self.type_info, - }, - )?; + s.serialize_field(&sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + type_info: self.type_info, + })?; } dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(v) => { - s.serialize_field( - DUMMY_FIELD_NAME, - &sequence::SequenceSerializeWrapper { - item_type: &v.value_type, - column: column.as_ref(), - type_info: self.type_info, - }, - )?; + s.serialize_field(&sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + type_info: self.type_info, + })?; } } } From 4b7c43c3230e22552e0b28700d081d5032eb0375 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 24 Jan 2024 12:24:40 +0100 Subject: [PATCH 087/104] Fix some unused code and import warnings --- libraries/extensions/ros2-bridge/python/src/typed/mod.rs | 1 - .../extensions/ros2-bridge/python/src/typed/serialize/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index b8b76893..2b91d08b 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -22,4 +22,3 @@ pub struct TypeInfo<'a> { /// the CDR format of ROS2 does not encode struct or field /// names. const DUMMY_STRUCT_NAME: &str = "struct"; -const DUMMY_FIELD_NAME: &str = "field"; diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs index 7f38c36c..ce9480db 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -5,9 +5,9 @@ use dora_ros2_bridge_msg_gen::types::{ primitives::{GenericString, NestableType}, MemberType, }; -use serde::ser::{SerializeStruct, SerializeTupleStruct}; +use serde::ser::SerializeTupleStruct; -use super::{TypeInfo, DUMMY_FIELD_NAME, DUMMY_STRUCT_NAME}; +use super::{TypeInfo, DUMMY_STRUCT_NAME}; mod array; mod defaults; From efb6a51c2246699ef46fa959919bd12308bb4d0b Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 24 Jan 2024 12:26:02 +0100 Subject: [PATCH 088/104] Fix: Wrap deserialized sequences into list of length 1 Required because arrow uses column-oriented data format, which requires all struct fields to have length 1. --- .../python/src/typed/deserialize/mod.rs | 1 - .../python/src/typed/deserialize/sequence.rs | 38 ++++++++++++++----- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs index 13288aa5..db9249d1 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/mod.rs @@ -143,7 +143,6 @@ impl<'a, 'de> serde::de::Visitor<'de> for StructVisitor<'a> { })?; fields.push(( - // Recreate a new field as List(UInt8) can be converted to UInt8 Arc::new(Field::new(&member.name, value.data_type().clone(), true)), make_array(value), )); diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs index ee918ef3..ecf74de2 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs @@ -1,11 +1,14 @@ use arrow::{ - array::{ArrayData, BooleanBuilder, PrimitiveBuilder, StringBuilder}, - datatypes::{self, ArrowPrimitiveType}, + array::{ + Array, ArrayData, BooleanBuilder, ListArray, ListBuilder, PrimitiveBuilder, StringBuilder, + }, + buffer::OffsetBuffer, + datatypes::{self, ArrowPrimitiveType, Field}, }; use core::fmt; use dora_ros2_bridge_msg_gen::types::primitives::{self, BasicType, NestableType}; use serde::Deserialize; -use std::{borrow::Cow, ops::Deref}; +use std::{borrow::Cow, ops::Deref, sync::Arc}; use crate::typed::TypeInfo; @@ -65,7 +68,10 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { while let Some(value) = seq.next_element()? { array.append_value(value); } - Ok(array.finish().into()) + // wrap array into list of length 1 + let mut list = ListBuilder::new(array); + list.append(true); + Ok(list.finish().into()) } }, NestableType::NamedType(name) => { @@ -99,7 +105,10 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { while let Some(value) = seq.next_element::()? { array.append_value(value); } - Ok(array.finish().into()) + // wrap array into list of length 1 + let mut list = ListBuilder::new(array); + list.append(true); + Ok(list.finish().into()) } primitives::GenericString::WString => todo!("deserialize sequence of WString"), primitives::GenericString::BoundedWString(_) => { @@ -122,9 +131,17 @@ where values.push(arrow::array::make_array(value)); } let refs: Vec<_> = values.iter().map(|a| a.deref()).collect(); - arrow::compute::concat(&refs) - .map(|a| a.to_data()) - .map_err(super::error) + let concatenated = arrow::compute::concat(&refs).map_err(super::error)?; + + let list = ListArray::try_new( + Arc::new(Field::new("item", concatenated.data_type().clone(), true)), + OffsetBuffer::from_lengths([concatenated.len()]), + Arc::new(concatenated), + None, + ) + .map_err(error)?; + + Ok(list.to_data()) } fn deserialize_primitive_seq<'de, S, T>( @@ -139,5 +156,8 @@ where while let Some(value) = seq.next_element::()? { array.append_value(value); } - Ok(array.finish().into()) + // wrap array into list of length 1 + let mut list = ListBuilder::new(array); + list.append(true); + Ok(list.finish().into()) } From 9eecf22fee5b081eaf559032b962eb751b687635 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 24 Jan 2024 12:26:19 +0100 Subject: [PATCH 089/104] Rename method --- .../ros2-bridge/python/src/typed/deserialize/sequence.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs index ecf74de2..a5592196 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/deserialize/sequence.rs @@ -82,7 +82,7 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { messages: self.type_info.messages.clone(), }), }; - deserialize_struct_array(&mut seq, deserializer) + deserialize_struct_seq(&mut seq, deserializer) } NestableType::NamespacedType(reference) => { if reference.namespace != "msg" { @@ -97,7 +97,7 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { messages: self.type_info.messages.clone(), }), }; - deserialize_struct_array(&mut seq, deserializer) + deserialize_struct_seq(&mut seq, deserializer) } NestableType::GenericString(t) => match t { primitives::GenericString::String | primitives::GenericString::BoundedString(_) => { @@ -119,7 +119,7 @@ impl<'de> serde::de::Visitor<'de> for SequenceVisitor<'_> { } } -fn deserialize_struct_array<'de, A>( +fn deserialize_struct_seq<'de, A>( seq: &mut A, deserializer: StructDeserializer<'_>, ) -> Result>::Error> From cf7c5552d15575eded46a07df72b6056e6ba187b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 12:17:32 +0100 Subject: [PATCH 090/104] Using python file to declare test array and test them --- Cargo.lock | 20 +- .../extensions/ros2-bridge/python/Cargo.toml | 3 + .../ros2-bridge/python/src/typed/mod.rs | 96 ++++++ .../ros2-bridge/python/test_utils.py | 284 ++++++++++++++++++ 4 files changed, 398 insertions(+), 5 deletions(-) create mode 100644 libraries/extensions/ros2-bridge/python/test_utils.py diff --git a/Cargo.lock b/Cargo.lock index af024277..ddb01497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1424,7 +1424,7 @@ dependencies = [ "notify", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "termcolor", "tokio", "tokio-stream", @@ -1461,7 +1461,7 @@ dependencies = [ "once_cell", "serde", "serde-with-expand-env", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "tokio", "tracing", "uuid", @@ -1727,6 +1727,7 @@ dependencies = [ "futures", "pyo3", "serde", + "serde_assert", ] [[package]] @@ -4976,6 +4977,15 @@ dependencies = [ "shellexpand 2.1.2", ] +[[package]] +name = "serde_assert" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92b7be0ad5a7b2eefaa5418eb141838270f1ad2d2c6e88acec3795d2425ffa97" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.195" @@ -5035,9 +5045,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap 2.0.2", "itoa", @@ -6626,7 +6636,7 @@ dependencies = [ "num_cpus", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "validated_struct", "zenoh-cfg-properties", "zenoh-core", diff --git a/libraries/extensions/ros2-bridge/python/Cargo.toml b/libraries/extensions/ros2-bridge/python/Cargo.toml index 5c80cc9b..fd1b8627 100644 --- a/libraries/extensions/ros2-bridge/python/Cargo.toml +++ b/libraries/extensions/ros2-bridge/python/Cargo.toml @@ -12,3 +12,6 @@ eyre = "0.6" serde = "1.0.166" arrow = { workspace = true, features = ["pyarrow"] } futures = "0.3.28" + +[dev-dependencies] +serde_assert = "0.7.1" diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index 2b91d08b..c6875b7f 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -22,3 +22,99 @@ pub struct TypeInfo<'a> { /// the CDR format of ROS2 does not encode struct or field /// names. const DUMMY_STRUCT_NAME: &str = "struct"; + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::typed::deserialize::StructDeserializer; + use crate::typed::serialize; + use crate::typed::TypeInfo; + use crate::Ros2Context; + + use arrow::array::make_array; + use arrow::pyarrow::FromPyArrow; + use arrow::pyarrow::ToPyArrow; + + use pyo3::types::IntoPyDict; + use pyo3::types::PyDict; + use pyo3::types::PyList; + use pyo3::types::PyModule; + use pyo3::types::PyTuple; + use pyo3::Python; + use serde::de::DeserializeSeed; + use serde::Serialize; + + use serde_assert::Serializer; + use serialize::TypedValue; + + use eyre::{Context, Result}; + use serde_assert::Deserializer; + #[test] + fn test_python_array_code() -> Result<()> { + pyo3::prepare_freethreaded_python(); + let context = Ros2Context::new(None).context("Could not create a context")?; + let messages = context.messages.clone(); + let serializer = Serializer::builder().build(); + + Python::with_gil(|py| -> Result<()> { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); //.join("test_utils.py"); // Adjust this path as needed + + // Add the Python module's directory to sys.path + py.run( + "import sys; sys.path.append(str(path))", + Some([("path", path)].into_py_dict(py)), + None, + )?; + + let my_module = PyModule::import(py, "test_utils")?; + + let arrays: &PyList = my_module.getattr("TEST_ARRAYS")?.extract()?; + for array_wrapper in arrays.iter() { + let arrays: &PyTuple = array_wrapper.extract()?; + let package_name: String = arrays.get_item(0)?.extract()?; + let message_name: String = arrays.get_item(1)?.extract()?; + println!("Checking {}::{}", package_name, message_name); + let in_pyarrow = arrays.get_item(2)?; + + let array = arrow::array::ArrayData::from_pyarrow(in_pyarrow)?; + let type_info = TypeInfo { + package_name: package_name.into(), + message_name: message_name.clone().into(), + messages: messages.clone(), + }; + let typed_value = TypedValue { + value: &make_array(array.clone()), + type_info: &type_info.clone(), + }; + + let typed_deserializer = + StructDeserializer::new(std::borrow::Cow::Owned(type_info)); + let tokens = typed_value.serialize(&serializer)?; + let mut deserializer = Deserializer::builder(tokens).build(); + + let out_value = typed_deserializer + .deserialize(&mut deserializer) + .context("could not deserialize array")?; + + let out_pyarrow = out_value.to_pyarrow(py)?; + + let test_utils = PyModule::import(py, "test_utils")?; + let context = PyDict::new(py); + + context.set_item("test_utils", test_utils)?; + context.set_item("in_pyarrow", in_pyarrow)?; + context.set_item("out_pyarrow", out_pyarrow)?; + + let _ = py + .eval( + "test_utils.is_subset(in_pyarrow, out_pyarrow)", + Some(context), + None, + ) + .context("could not check if it is a subset")?; + } + Ok(()) + }) + } +} diff --git a/libraries/extensions/ros2-bridge/python/test_utils.py b/libraries/extensions/ros2-bridge/python/test_utils.py new file mode 100644 index 00000000..a0dd5e6f --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/test_utils.py @@ -0,0 +1,284 @@ +import numpy as np +import pyarrow as pa + + +# Marker Message Example +TEST_ARRAYS = [ + ("std_msgs", "UInt8", pa.array([{"data": np.uint8(2)}])), + ( + "std_msgs", + "String", + pa.array([{"data": "hello"}]), + ), + ( + "std_msgs", + "UInt8MultiArray", + pa.array( + [ + { + "data": np.array([1, 2, 3, 4], np.uint8), + "layout": { + "dim": [ + { + "label": "a", + "size": np.uint32(10), + "stride": np.uint32(20), + } + ], + "data_offset": np.uint32(30), + }, + } + ] + ), + ), + ( + "std_msgs", + "Float32MultiArray", + pa.array( + [ + { + "data": np.array([1, 2, 3, 4], np.float32), + "layout": { + "dim": [ + { + "label": "a", + "size": np.uint32(10), + "stride": np.uint32(20), + } + ], + "data_offset": np.uint32(30), + }, + } + ] + ), + ), + ( + "visualization_msgs", + "Marker", + pa.array( + [ + { + "header": { + "frame_id": "world", # Placeholder value (String type, no numpy equivalent) + }, + "ns": "my_namespace", # Placeholder value (String type, no numpy equivalent) + "id": np.int32(1), # Numpy type + "type": np.int32(0), # Numpy type (ARROW) + "action": np.int32(0), # Numpy type (ADD) + "lifetime": { + "sec": np.int32(1), + "nanosec": np.uint32(2), + }, # Numpy type + "pose": { + "position": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(2.0), # Numpy type + "z": np.float64(3.0), # Numpy type + }, + "orientation": { + "x": np.float64(0.0), # Numpy type + "y": np.float64(0.0), # Numpy type + "z": np.float64(0.0), # Numpy type + "w": np.float64(1.0), # Numpy type + }, + }, + "scale": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + }, + "color": { + "r": np.float32(1.0), # Numpy type + "g": np.float32(0.0), # Numpy type + "b": np.float32(0.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + }, + "frame_locked": False, # Boolean type, no numpy equivalent + "points": [ # Numpy array for points + { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + } + ], + "colors": [ + { + "r": np.float32(1.0), # Numpy type + "g": np.float32(1.0), # Numpy type + "b": np.float32(1.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + } # Numpy array for colors + ], + "texture_resource": "", + "uv_coordinates": [{}], + "text": "", + "mesh_resource": "", + "mesh_use_embedded_materials": False, # Boolean type, no numpy equivalent + } + ] + ), + ), + ( + "visualization_msgs", + "MarkerArray", + pa.array( + [ + { + "markers": [ + { + "header": { + "frame_id": "world", # Placeholder value (String type, no numpy equivalent) + }, + "ns": "my_namespace", # Placeholder value (String type, no numpy equivalent) + "id": np.int32(1), # Numpy type + "type": np.int32(0), # Numpy type (ARROW) + "action": np.int32(0), # Numpy type (ADD) + "lifetime": { + "sec": np.int32(1), + "nanosec": np.uint32(2), + }, # Numpy type + "pose": { + "position": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(2.0), # Numpy type + "z": np.float64(3.0), # Numpy type + }, + "orientation": { + "x": np.float64(0.0), # Numpy type + "y": np.float64(0.0), # Numpy type + "z": np.float64(0.0), # Numpy type + "w": np.float64(1.0), # Numpy type + }, + }, + "scale": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + }, + "color": { + "r": np.float32(1.0), # Numpy type + "g": np.float32(0.0), # Numpy type + "b": np.float32(0.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + }, + "frame_locked": False, # Boolean type, no numpy equivalent + "points": [ # Numpy array for points + { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + } + ], + "colors": [ + { + "r": np.float32(1.0), # Numpy type + "g": np.float32(1.0), # Numpy type + "b": np.float32(1.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + } # Numpy array for colors + ], + "texture_resource": "", + "uv_coordinates": [{}], + "text": "", + "mesh_resource": "", + "mesh_use_embedded_materials": False, # Boolean type, no numpy equivalent + } + ] + } + ] + ), + ), + ( + "visualization_msgs", + "ImageMarker", + pa.array( + [ + { + "header": { + "stamp": { + "sec": np.int32(123456), # 32-bit integer + "nanosec": np.uint32(789), # 32-bit unsigned integer + }, + "frame_id": "frame_example", + }, + "ns": "namespace", + "id": np.int32(1), # 32-bit integer + "type": np.int32(0), # 32-bit integer, e.g., CIRCLE = 0 + "action": np.int32(0), # 32-bit integer, e.g., ADD = 0 + "position": { + "x": np.float64(1.0), # 32-bit float + "y": np.float64(2.0), # 32-bit float + "z": np.float64(3.0), # 32-bit float + }, + "scale": np.float32(1.0), # 32-bit float + "outline_color": { + "r": np.float32(255.0), # 32-bit float + "g": np.float32(0.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + "filled": np.uint8(1), # 8-bit unsigned integer + "fill_color": { + "r": np.float32(0.0), # 32-bit float + "g": np.float32(255.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + "lifetime": { + "sec": np.int32(300), # 32-bit integer + "nanosec": np.uint32(0), # 32-bit unsigned integer + }, + "points": [ + { + "x": np.float64(1.0), # 32-bit float + "y": np.float64(2.0), # 32-bit float + "z": np.float64(3.0), # 32-bit float + }, + { + "x": np.float64(4.0), # 32-bit float + "y": np.float64(5.0), # 32-bit float + "z": np.float64(6.0), # 32-bit float + }, + ], + "outline_colors": [ + { + "r": np.float32(255.0), # 32-bit float + "g": np.float32(0.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + { + "r": np.float32(0.0), # 32-bit float + "g": np.float32(255.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + ], + } + ] + ), + ), +] + + +def is_subset(subset, superset): + """ + Check if subset is a subset of superset, to avoid false negatives linked to default values. + """ + if isinstance(subset, pa.Array): + return is_subset(subset.to_pylist(), superset.to_pylist()) + + match subset: + case dict(_): + return all( + key in superset and is_subset(val, superset[key]) + for key, val in subset.items() + ) + case list(_) | set(_): + return all( + any(is_subset(subitem, superitem) for superitem in superset) + for subitem in subset + ) + # assume that subset is a plain value if none of the above match + case _: + return subset == superset From c9e4a696a3b9a1422fed34fbb70a04247ade0b82 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 12:23:48 +0100 Subject: [PATCH 091/104] Adding numpy and pyarrow dependencies --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 207a5c12..345edeca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,8 @@ jobs: run: cargo check --all - name: "Build (Without Python node as it is build with maturin)" run: cargo build --all --exclude dora-node-api-python + - name: "Install pyarrow for testing" + run: pip install numpy pyarrow - name: "Test" run: cargo test --all From 1e510a4236fdbc223659028aabf3557f3e283ce7 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 25 Jan 2024 13:10:16 +0100 Subject: [PATCH 092/104] Improve error message: print message type and value Co-authored-by: Haixuan Xavier Tao --- .../ros2-bridge/python/src/typed/serialize/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs index ce9480db..f4cee1b7 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -40,10 +40,12 @@ impl serde::Serialize for TypedValue<'_> { )) })?; - let input = self - .value - .as_struct_opt() - .ok_or_else(|| error("expected struct array"))?; + let input = self.value.as_struct_opt().ok_or_else(|| { + error(format!( + "expected struct array for message: {}, with following format: {:#?} \n But, got value: {:#?}", + self.type_info.message_name, message, self.value + )) + })?; for column_name in input.column_names() { if !message.members.iter().any(|m| m.name == column_name) { return Err(error(format!( From c87886296676ddd28eff9213d3b09370cbd11bd3 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 13:52:33 +0100 Subject: [PATCH 093/104] Use forward slash as it is default way of defining ros2 topic --- examples/python-ros2-dataflow/random_turtle.py | 6 ++---- libraries/extensions/ros2-bridge/python/src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/python-ros2-dataflow/random_turtle.py b/examples/python-ros2-dataflow/random_turtle.py index 3e97228b..e6fa2c75 100755 --- a/examples/python-ros2-dataflow/random_turtle.py +++ b/examples/python-ros2-dataflow/random_turtle.py @@ -21,14 +21,12 @@ topic_qos = dora.experimental.ros2_bridge.Ros2QosPolicies( # Create a publisher to cmd_vel topic turtle_twist_topic = ros2_node.create_topic( - "/turtle1/cmd_vel", "geometry_msgs::Twist", topic_qos + "/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos ) twist_writer = ros2_node.create_publisher(turtle_twist_topic) # Create a listener to pose topic -turtle_pose_topic = ros2_node.create_topic( - "/turtle1/pose", "turtlesim::Pose", topic_qos -) +turtle_pose_topic = ros2_node.create_topic("/turtle1/pose", "turtlesim/Pose", topic_qos) pose_reader = ros2_node.create_subscription(turtle_pose_topic) # Create a dora node diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index 9d06bbaf..f072ceb8 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -101,7 +101,7 @@ impl Ros2Node { message_type: String, qos: qos::Ros2QosPolicies, ) -> eyre::Result { - let (namespace_name, message_name) = message_type.split_once("::").with_context(|| { + let (namespace_name, message_name) = message_type.split_once("/").with_context(|| { format!( "message type must be of form `package::type`, is `{}`", message_type From aa2ffafba18c033b7e48b4be0a6c068365434cba Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 13:58:32 +0100 Subject: [PATCH 094/104] Test ros2-bridge within the ros2 job to use the right dependencies --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345edeca..aad0b15a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,10 +64,8 @@ jobs: run: cargo check --all - name: "Build (Without Python node as it is build with maturin)" run: cargo build --all --exclude dora-node-api-python - - name: "Install pyarrow for testing" - run: pip install numpy pyarrow - name: "Test" - run: cargo test --all + run: cargo test --all --exclude dora-ros2-bridge-python # Run examples as separate job because otherwise we will exhaust the disk # space of the GitHub action runners. @@ -166,6 +164,10 @@ jobs: with: required-ros-distributions: humble - run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"' + - name: "Install pyarrow for testing" + run: pip install numpy pyarrow + - name: "Test" + run: cargo test -p dora-ros2-bridge-python - name: "Rust ROS2 Bridge example" timeout-minutes: 30 env: From 1addfa385a4815a32e817423f99a933f403a4fc1 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 25 Jan 2024 16:22:45 +0100 Subject: [PATCH 095/104] Add more context to error messages when serializing nested message --- .../python/src/typed/serialize/mod.rs | 191 ++++++++++-------- 1 file changed, 111 insertions(+), 80 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs index f4cee1b7..8420f14f 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/serialize/mod.rs @@ -1,10 +1,14 @@ use std::{borrow::Cow, collections::HashMap, fmt::Display}; -use arrow::array::{Array, ArrayRef, AsArray}; +use arrow::{ + array::{Array, ArrayRef, AsArray}, + error, +}; use dora_ros2_bridge_msg_gen::types::{ primitives::{GenericString, NestableType}, MemberType, }; +use eyre::Context; use serde::ser::SerializeTupleStruct; use super::{TypeInfo, DUMMY_STRUCT_NAME}; @@ -42,8 +46,8 @@ impl serde::Serialize for TypedValue<'_> { let input = self.value.as_struct_opt().ok_or_else(|| { error(format!( - "expected struct array for message: {}, with following format: {:#?} \n But, got value: {:#?}", - self.type_info.message_name, message, self.value + "expected struct array for message: {}, with following format: {:#?} \n But, got type: {:#?}", + self.type_info.message_name, message, self.value.data_type() )) })?; for column_name in input.column_names() { @@ -73,95 +77,122 @@ impl serde::Serialize for TypedValue<'_> { &self.type_info.package_name, &self.type_info.messages, ) - .map_err(error)?; + .with_context(|| { + format!( + "failed to calculate default value for field {}.{}", + message.name, field.name + ) + }) + .map_err(|e| error(format!("{e:?}")))?; Cow::Owned(arrow::array::make_array(default)) } }; - match &field.r#type { - MemberType::NestableType(t) => match t { - NestableType::BasicType(t) => { - s.serialize_field(&primitive::SerializeWrapper { - t, - column: column.as_ref(), - })?; - } - NestableType::NamedType(name) => { - let referenced_value = &TypedValue { - value: column.as_ref(), - type_info: &TypeInfo { - package_name: Cow::Borrowed(&self.type_info.package_name), - message_name: Cow::Borrowed(&name.0), - messages: self.type_info.messages.clone(), - }, - }; - s.serialize_field(&referenced_value)?; - } - NestableType::NamespacedType(reference) => { - if reference.namespace != "msg" { - return Err(error(format!( - "struct field {} references non-message type {reference:?}", - field.name - ))); - } + self.serialize_field::(field, column, &mut s) + .map_err(|e| { + error(format!( + "failed to serialize field {}.{}: {e}", + message.name, field.name + )) + })?; + } + s.end() + } +} - let referenced_value: &TypedValue<'_> = &TypedValue { - value: column.as_ref(), - type_info: &TypeInfo { - package_name: Cow::Borrowed(&reference.package), - message_name: Cow::Borrowed(&reference.name), - messages: self.type_info.messages.clone(), - }, - }; - s.serialize_field(&referenced_value)?; - } - NestableType::GenericString(t) => match t { - GenericString::String | GenericString::BoundedString(_) => { - let string = if let Some(string_array) = column.as_string_opt::() { - // should match the length of the outer struct array - assert_eq!(string_array.len(), 1); - string_array.value(0) - } else { - // try again with large offset type - let string_array = column - .as_string_opt::() - .ok_or_else(|| error("expected string array"))?; - // should match the length of the outer struct array - assert_eq!(string_array.len(), 1); - string_array.value(0) - }; - s.serialize_field(string)?; - } - GenericString::WString => todo!("serializing WString types"), - GenericString::BoundedWString(_) => { - todo!("serializing BoundedWString types") - } - }, - }, - dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { - s.serialize_field(&array::ArraySerializeWrapper { - array_info: a, +impl<'a> TypedValue<'a> { + fn serialize_field( + &self, + field: &dora_ros2_bridge_msg_gen::types::Member, + column: Cow<'_, std::sync::Arc>, + s: &mut S::SerializeTupleStruct, + ) -> Result<(), S::Error> + where + S: serde::Serializer, + { + match &field.r#type { + MemberType::NestableType(t) => match t { + NestableType::BasicType(t) => { + s.serialize_field(&primitive::SerializeWrapper { + t, column: column.as_ref(), - type_info: self.type_info, })?; } - dora_ros2_bridge_msg_gen::types::MemberType::Sequence(v) => { - s.serialize_field(&sequence::SequenceSerializeWrapper { - item_type: &v.value_type, - column: column.as_ref(), - type_info: self.type_info, - })?; + NestableType::NamedType(name) => { + let referenced_value = &TypedValue { + value: column.as_ref(), + type_info: &TypeInfo { + package_name: Cow::Borrowed(&self.type_info.package_name), + message_name: Cow::Borrowed(&name.0), + messages: self.type_info.messages.clone(), + }, + }; + s.serialize_field(&referenced_value)?; } - dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(v) => { - s.serialize_field(&sequence::SequenceSerializeWrapper { - item_type: &v.value_type, - column: column.as_ref(), - type_info: self.type_info, - })?; + NestableType::NamespacedType(reference) => { + if reference.namespace != "msg" { + return Err(error(format!( + "struct field {} references non-message type {reference:?}", + field.name + ))); + } + + let referenced_value: &TypedValue<'_> = &TypedValue { + value: column.as_ref(), + type_info: &TypeInfo { + package_name: Cow::Borrowed(&reference.package), + message_name: Cow::Borrowed(&reference.name), + messages: self.type_info.messages.clone(), + }, + }; + s.serialize_field(&referenced_value)?; } + NestableType::GenericString(t) => match t { + GenericString::String | GenericString::BoundedString(_) => { + let string = if let Some(string_array) = column.as_string_opt::() { + // should match the length of the outer struct array + assert_eq!(string_array.len(), 1); + string_array.value(0) + } else { + // try again with large offset type + let string_array = column + .as_string_opt::() + .ok_or_else(|| error("expected string array"))?; + // should match the length of the outer struct array + assert_eq!(string_array.len(), 1); + string_array.value(0) + }; + s.serialize_field(string)?; + } + GenericString::WString => todo!("serializing WString types"), + GenericString::BoundedWString(_) => { + todo!("serializing BoundedWString types") + } + }, + }, + dora_ros2_bridge_msg_gen::types::MemberType::Array(a) => { + s.serialize_field(&array::ArraySerializeWrapper { + array_info: a, + column: column.as_ref(), + type_info: self.type_info, + })?; + } + dora_ros2_bridge_msg_gen::types::MemberType::Sequence(v) => { + s.serialize_field(&sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + type_info: self.type_info, + })?; + } + dora_ros2_bridge_msg_gen::types::MemberType::BoundedSequence(v) => { + s.serialize_field(&sequence::SequenceSerializeWrapper { + item_type: &v.value_type, + column: column.as_ref(), + type_info: self.type_info, + })?; } } - s.end() + Ok(()) } } From e12b29dcc6b16aba72b0a33db6cd5d7bf1aba3db Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 26 Jan 2024 09:57:00 +0100 Subject: [PATCH 096/104] Make type setting available for both forward slash and `::` --- libraries/extensions/ros2-bridge/python/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/extensions/ros2-bridge/python/src/lib.rs b/libraries/extensions/ros2-bridge/python/src/lib.rs index f072ceb8..c9492dca 100644 --- a/libraries/extensions/ros2-bridge/python/src/lib.rs +++ b/libraries/extensions/ros2-bridge/python/src/lib.rs @@ -101,12 +101,13 @@ impl Ros2Node { message_type: String, qos: qos::Ros2QosPolicies, ) -> eyre::Result { - let (namespace_name, message_name) = message_type.split_once("/").with_context(|| { - format!( - "message type must be of form `package::type`, is `{}`", - message_type - ) - })?; + let (namespace_name, message_name) = + match (message_type.split_once("/"), message_type.split_once("::")) { + (Some(msg), None) => msg, + (None, Some(msg)) => msg, + _ => eyre::bail!("Expected message type in the format `namespace/message` or `namespace::message`, such as `std_msgs/UInt8` but got: {}", message_type), + }; + let message_type_name = ros2_client::MessageTypeName::new(namespace_name, message_name); let topic_name = ros2_client::Name::parse(name) .map_err(|err| eyre!("failed to parse ROS2 topic name: {err}"))?; From cb74ee1aeefbe17e079acd663c256c315faaa9b9 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 26 Jan 2024 12:58:07 +0100 Subject: [PATCH 097/104] Bump to `0.3.2-rc1` --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- Changelog.md | 15 +++++++ README.md | 14 +++---- apis/python/node/dora/__init__.py | 2 +- 5 files changed, 75 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d3319a6..351f73b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "flume", "zenoh", @@ -1015,7 +1015,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.1" +version = "0.3.2-rc1" [[package]] name = "concurrent-queue" @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "arrow", "eyre", @@ -1405,7 +1405,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "ctrlc", "dora-core", @@ -1453,7 +1453,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "aligned-vec", "dora-message", @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "aligned-vec", "async-trait", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "eyre", "reqwest", @@ -1523,7 +1523,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "arrow-data", "arrow-schema", @@ -1534,7 +1534,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "opentelemetry 0.21.0", "opentelemetry-otlp", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "aligned-vec", "arrow", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "arrow-array", "dora-node-api", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "cxx", "cxx-build", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "arrow", "dora-node-api", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1611,14 +1611,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "cxx", "cxx-build", @@ -1627,7 +1627,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "proc-macro2", "quote", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "aligned-vec", "arrow", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "arrow", "dora-arrow-convert", @@ -1659,7 +1659,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "chrono", "dora-node-api", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "aligned-vec", "arrow", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3239,7 +3239,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -3250,14 +3250,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -4658,7 +4658,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -4669,14 +4669,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "eyre", @@ -4684,7 +4684,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5088,7 +5088,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.1" +version = "0.3.2-rc1" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 4d533ed6..ff7b0740 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.1" +version = "0.3.2-rc1" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.1", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.1", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.1", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.1", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.1", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.1", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.1", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.1", path = "apis/c/node" } -dora-core = { version = "0.3.1", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.1", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.1", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.1", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.1", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.1", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.1", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.1", path = "libraries/message" } -dora-runtime = { version = "0.3.1", path = "binaries/runtime" } -dora-daemon = { version = "0.3.1", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.1", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.2-rc1", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.2-rc1", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.2-rc1", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.2-rc1", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.2-rc1", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.2-rc1", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.2-rc1", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.2-rc1", path = "apis/c/node" } +dora-core = { version = "0.3.2-rc1", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.2-rc1", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.2-rc1", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.2-rc1", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.2-rc1", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.2-rc1", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.2-rc1", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.2-rc1", path = "libraries/message" } +dora-runtime = { version = "0.3.2-rc1", path = "binaries/runtime" } +dora-daemon = { version = "0.3.2-rc1", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.2-rc1", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/Changelog.md b/Changelog.md index ee0fc100..7fc2b799 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,20 @@ # Changelog +## v0.3.2 (2024-01-26) + +## Features + +- Wait until `DestroyResult` is sent before exiting dora-daemon by @phil-opp in https://github.com/dora-rs/dora/pull/413 +- Reduce dora-rs to a single binary by @haixuanTao in https://github.com/dora-rs/dora/pull/410 +- Rework python ROS2 (de)serialization using parsed ROS2 messages directly by @phil-opp in https://github.com/dora-rs/dora/pull/415 +- Fix ros2 array bug by @haixuanTao in https://github.com/dora-rs/dora/pull/412 +- Test ros2 type info by @haixuanTao in https://github.com/dora-rs/dora/pull/418 +- Use forward slash as it is default way of defining ros2 topic by @haixuanTao in https://github.com/dora-rs/dora/pull/419 + +## Minor + +- Bump h2 from 0.3.21 to 0.3.24 by @dependabot in https://github.com/dora-rs/dora/pull/414 + ## v0.3.1 (2024-01-09) ## Features diff --git a/README.md b/README.md index a51d2fcc..1c7b9905 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,6 @@ Quickest way: ```bash cargo install dora-cli -cargo install dora-coordinator -cargo install dora-daemon pip install dora-rs ## For Python API dora --help @@ -71,17 +69,17 @@ For more installation guideline, check out our installation guide here: https:// 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.1/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index c5351f50..7e8fbbc6 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.1" +__version__ = "0.3.2-rc1" class DoraStatus(Enum): From ead8eb965454b151871587ce4bc1a53f7994b9f1 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 26 Jan 2024 18:14:08 +0100 Subject: [PATCH 098/104] Fix release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43c31bdc..d4cbfd7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,10 +57,10 @@ jobs: cargo publish -p dora-node-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Publish binaries crates - cargo publish -p dora-cli --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-coordinator --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-runtime --token ${{ secrets.CARGO_REGISTRY_TOKEN }} cargo publish -p dora-daemon --token ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish -p dora-cli --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Publish extension crates cargo publish -p dora-record --token ${{ secrets.CARGO_REGISTRY_TOKEN }} From 573a7bc425eec3a4d648c33c1862b8a80bf1b0e1 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 26 Jan 2024 18:02:32 +0100 Subject: [PATCH 099/104] Make it possible to check the logs without passing uuid This commit make it possible to check the logs as follows: ```bash dora logs object_detection ``` If more than one dataflow is running, we will inquire for which dataflow to use. --- binaries/cli/src/main.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index ab6383bf..4b25cbdd 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -90,7 +90,10 @@ enum Command { // Planned for future releases: // Dashboard, /// Show logs of a given dataflow and node. - Logs { dataflow: String, node: String }, + Logs { + dataflow_or_node: String, + node: Option, + }, // Metrics, // Stats, // Get, @@ -191,12 +194,29 @@ fn run() -> eyre::Result<()> { internal_create_with_path_dependencies, } => template::create(args, internal_create_with_path_dependencies)?, Command::Up { config } => up::up(config.as_deref())?, - Command::Logs { dataflow, node } => { - let uuid = Uuid::parse_str(&dataflow).ok(); - let name = if uuid.is_some() { None } else { Some(dataflow) }; + + Command::Logs { + dataflow_or_node, + node, + } => { let mut session = connect_to_coordinator().wrap_err("failed to connect to dora coordinator")?; - logs::logs(&mut *session, uuid, name, node)? + let uuids = query_running_dataflows(&mut *session) + .wrap_err("failed to query running dataflows")?; + if let Some(node) = node { + let dataflow = dataflow_or_node; + let uuid = Uuid::parse_str(&dataflow).ok(); + let name = if uuid.is_some() { None } else { Some(dataflow) }; + logs::logs(&mut *session, uuid, name, node)? + } else { + let uuid = match &uuids[..] { + [] => bail!("No dataflows are running"), + [uuid] => uuid.clone(), + _ => inquire::Select::new("Choose dataflow to show logs:", uuids).prompt()?, + }; + let node = dataflow_or_node; + logs::logs(&mut *session, Some(uuid.uuid), None, node)? + } } Command::Start { dataflow, From 0985bd4f9bf091997850c0695ca50ed3e548cb0a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Jan 2024 10:29:58 +0100 Subject: [PATCH 100/104] Make first positional argument optional Bump 0.3.2-rc2 --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- README.md | 12 +++--- apis/python/node/dora/__init__.py | 2 +- binaries/cli/src/main.rs | 14 +++---- 5 files changed, 65 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 351f73b7..d7f2f8a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "flume", "zenoh", @@ -1015,7 +1015,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.2-rc1" +version = "0.3.2-rc2" [[package]] name = "concurrent-queue" @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "arrow", "eyre", @@ -1405,7 +1405,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "ctrlc", "dora-core", @@ -1453,7 +1453,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "aligned-vec", "dora-message", @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "aligned-vec", "async-trait", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "eyre", "reqwest", @@ -1523,7 +1523,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "arrow-data", "arrow-schema", @@ -1534,7 +1534,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "opentelemetry 0.21.0", "opentelemetry-otlp", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "arrow-array", "dora-node-api", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "cxx", "cxx-build", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "arrow", "dora-node-api", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1611,14 +1611,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "cxx", "cxx-build", @@ -1627,7 +1627,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "proc-macro2", "quote", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "arrow", "dora-arrow-convert", @@ -1659,7 +1659,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "chrono", "dora-node-api", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "aligned-vec", "arrow", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3239,7 +3239,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -3250,14 +3250,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4658,7 +4658,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4669,14 +4669,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "eyre", @@ -4684,7 +4684,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5088,7 +5088,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.2-rc1" +version = "0.3.2-rc2" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index ff7b0740..34dcaacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.2-rc1" +version = "0.3.2-rc2" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.2-rc1", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.2-rc1", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.2-rc1", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.2-rc1", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.2-rc1", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.2-rc1", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.2-rc1", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.2-rc1", path = "apis/c/node" } -dora-core = { version = "0.3.2-rc1", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.2-rc1", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.2-rc1", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.2-rc1", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.2-rc1", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.2-rc1", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.2-rc1", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.2-rc1", path = "libraries/message" } -dora-runtime = { version = "0.3.2-rc1", path = "binaries/runtime" } -dora-daemon = { version = "0.3.2-rc1", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.2-rc1", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.2-rc2", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.2-rc2", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.2-rc2", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.2-rc2", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.2-rc2", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.2-rc2", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.2-rc2", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.2-rc2", path = "apis/c/node" } +dora-core = { version = "0.3.2-rc2", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.2-rc2", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.2-rc2", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.2-rc2", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.2-rc2", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.2-rc2", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.2-rc2", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.2-rc2", path = "libraries/message" } +dora-runtime = { version = "0.3.2-rc2", path = "binaries/runtime" } +dora-daemon = { version = "0.3.2-rc2", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.2-rc2", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/README.md b/README.md index 1c7b9905..32f5193c 100644 --- a/README.md +++ b/README.md @@ -69,17 +69,17 @@ For more installation guideline, check out our installation guide here: https:// 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc1/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 7e8fbbc6..d8ee3e67 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.2-rc1" +__version__ = "0.3.2-rc2" class DoraStatus(Enum): diff --git a/binaries/cli/src/main.rs b/binaries/cli/src/main.rs index 4b25cbdd..e6b5f757 100644 --- a/binaries/cli/src/main.rs +++ b/binaries/cli/src/main.rs @@ -90,9 +90,10 @@ enum Command { // Planned for future releases: // Dashboard, /// Show logs of a given dataflow and node. + #[command(allow_missing_positional = true)] Logs { - dataflow_or_node: String, - node: Option, + dataflow: Option, + node: String, }, // Metrics, // Stats, @@ -195,16 +196,12 @@ fn run() -> eyre::Result<()> { } => template::create(args, internal_create_with_path_dependencies)?, Command::Up { config } => up::up(config.as_deref())?, - Command::Logs { - dataflow_or_node, - node, - } => { + Command::Logs { dataflow, node } => { let mut session = connect_to_coordinator().wrap_err("failed to connect to dora coordinator")?; let uuids = query_running_dataflows(&mut *session) .wrap_err("failed to query running dataflows")?; - if let Some(node) = node { - let dataflow = dataflow_or_node; + if let Some(dataflow) = dataflow { let uuid = Uuid::parse_str(&dataflow).ok(); let name = if uuid.is_some() { None } else { Some(dataflow) }; logs::logs(&mut *session, uuid, name, node)? @@ -214,7 +211,6 @@ fn run() -> eyre::Result<()> { [uuid] => uuid.clone(), _ => inquire::Select::new("Choose dataflow to show logs:", uuids).prompt()?, }; - let node = dataflow_or_node; logs::logs(&mut *session, Some(uuid.uuid), None, node)? } } From 2faed0e828579d399d915c6e11e232269b5e38a6 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 29 Jan 2024 11:42:03 +0100 Subject: [PATCH 101/104] Bump to v0.3.2 --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 40 +++++++++---------- README.md | 12 +++--- apis/python/node/dora/__init__.py | 2 +- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7f2f8a4..17b82a08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "benchmark-example-node" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "flume", "zenoh", @@ -1015,7 +1015,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.3.2-rc2" +version = "0.3.2" [[package]] name = "concurrent-queue" @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "dora-arrow-convert" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "arrow", "eyre", @@ -1405,7 +1405,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "bat", "clap 4.4.6", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "ctrlc", "dora-core", @@ -1453,7 +1453,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "aligned-vec", "dora-message", @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "aligned-vec", "async-trait", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "eyre", "reqwest", @@ -1523,7 +1523,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "arrow-data", "arrow-schema", @@ -1534,7 +1534,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "opentelemetry 0.21.0", "opentelemetry-otlp", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "aligned-vec", "arrow", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "arrow-array", "dora-node-api", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "cxx", "cxx-build", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "arrow", "dora-node-api", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-arrow-convert", "dora-operator-api-macros", @@ -1611,14 +1611,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "cxx", "cxx-build", @@ -1627,7 +1627,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "proc-macro2", "quote", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "aligned-vec", "arrow", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "arrow", "dora-arrow-convert", @@ -1659,7 +1659,7 @@ dependencies = [ [[package]] name = "dora-record" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "chrono", "dora-node-api", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "dora-runtime" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "aligned-vec", "arrow", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3239,7 +3239,7 @@ dependencies = [ [[package]] name = "multiple-daemons-example-node" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -3250,14 +3250,14 @@ dependencies = [ [[package]] name = "multiple-daemons-example-operator" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-operator-api", ] [[package]] name = "multiple-daemons-example-sink" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -4658,7 +4658,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -4669,14 +4669,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "eyre", @@ -4684,7 +4684,7 @@ dependencies = [ [[package]] name = "rust-ros2-dataflow-example-node" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "dora-node-api", "dora-ros2-bridge", @@ -5088,7 +5088,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.3.2-rc2" +version = "0.3.2" dependencies = [ "bincode", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 34dcaacf..6b27c18e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,31 +33,31 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.3.2-rc2" +version = "0.3.2" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.3.2-rc2", path = "apis/rust/node", default-features = false } -dora-node-api-python = { version = "0.3.2-rc2", path = "apis/python/node", default-features = false } -dora-operator-api = { version = "0.3.2-rc2", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.3.2-rc2", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.3.2-rc2", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.3.2-rc2", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.3.2-rc2", path = "apis/c/operator" } -dora-node-api-c = { version = "0.3.2-rc2", path = "apis/c/node" } -dora-core = { version = "0.3.2-rc2", path = "libraries/core" } -dora-arrow-convert = { version = "0.3.2-rc2", path = "libraries/arrow-convert" } -dora-tracing = { version = "0.3.2-rc2", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.3.2-rc2", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.3.2-rc2", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.3.2-rc2", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.3.2-rc2", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.3.2-rc2", path = "libraries/message" } -dora-runtime = { version = "0.3.2-rc2", path = "binaries/runtime" } -dora-daemon = { version = "0.3.2-rc2", path = "binaries/daemon" } -dora-coordinator = { version = "0.3.2-rc2", path = "binaries/coordinator" } +dora-node-api = { version = "0.3.2", path = "apis/rust/node", default-features = false } +dora-node-api-python = { version = "0.3.2", path = "apis/python/node", default-features = false } +dora-operator-api = { version = "0.3.2", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.3.2", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.3.2", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.3.2", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.3.2", path = "apis/c/operator" } +dora-node-api-c = { version = "0.3.2", path = "apis/c/node" } +dora-core = { version = "0.3.2", path = "libraries/core" } +dora-arrow-convert = { version = "0.3.2", path = "libraries/arrow-convert" } +dora-tracing = { version = "0.3.2", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.3.2", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.3.2", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.3.2", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.3.2", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.3.2", path = "libraries/message" } +dora-runtime = { version = "0.3.2", path = "binaries/runtime" } +dora-daemon = { version = "0.3.2", path = "binaries/daemon" } +dora-coordinator = { version = "0.3.2", path = "binaries/coordinator" } dora-ros2-bridge = { path = "libraries/extensions/ros2-bridge" } dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" } arrow = "48.0.0" diff --git a/README.md b/README.md index 32f5193c..c455cf54 100644 --- a/README.md +++ b/README.md @@ -69,17 +69,17 @@ For more installation guideline, check out our installation guide here: https:// 1. Install the example python dependencies: ```bash -pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/requirements.txt +pip install -r https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/requirements.txt ``` 2. Get some example operators: ```bash -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/webcam.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/plot.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/utils.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/object_detection.py -wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2-rc2/examples/python-operator-dataflow/dataflow.yml +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/webcam.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/plot.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/utils.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/object_detection.py +wget https://raw.githubusercontent.com/dora-rs/dora/v0.3.2/examples/python-operator-dataflow/dataflow.yml ``` 3. Start the dataflow diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index d8ee3e67..9c3143d2 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -15,7 +15,7 @@ from enum import Enum from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.3.2-rc2" +__version__ = "0.3.2" class DoraStatus(Enum): From 0725620eabe9d63db077b40d009d96eb6194776b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 2 Feb 2024 16:39:26 +0100 Subject: [PATCH 102/104] Simplyfing the metrics functionality by using meter provider instead of the global meter. --- Cargo.lock | 10 +++++----- apis/python/node/Cargo.toml | 2 +- binaries/runtime/src/lib.rs | 17 +++++++---------- .../extensions/telemetry/metrics/Cargo.toml | 3 ++- .../extensions/telemetry/metrics/src/lib.rs | 3 +++ 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b82a08..9ce8f9ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1538,7 +1538,7 @@ version = "0.3.2" dependencies = [ "opentelemetry 0.21.0", "opentelemetry-otlp", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", ] [[package]] @@ -3677,7 +3677,7 @@ dependencies = [ "opentelemetry 0.21.0", "opentelemetry-proto", "opentelemetry-semantic-conventions 0.13.0", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "prost", "thiserror", "tokio", @@ -3691,7 +3691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" dependencies = [ "opentelemetry 0.21.0", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "prost", "tonic", ] @@ -3768,9 +3768,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", diff --git a/apis/python/node/Cargo.toml b/apis/python/node/Cargo.toml index 3fbcccb8..6e05782a 100644 --- a/apis/python/node/Cargo.toml +++ b/apis/python/node/Cargo.toml @@ -20,7 +20,7 @@ pyo3 = { workspace = true, features = ["eyre", "abi3-py37"] } eyre = "0.6" serde_yaml = "0.8.23" flume = "0.10.14" -dora-runtime = { workspace = true, features = ["tracing", "python"] } +dora-runtime = { workspace = true, features = ["tracing", "metrics", "python"] } arrow = { workspace = true, features = ["pyarrow"] } pythonize = { workspace = true } futures = "0.3.28" diff --git a/binaries/runtime/src/lib.rs b/binaries/runtime/src/lib.rs index 9d903889..81ef6f5a 100644 --- a/binaries/runtime/src/lib.rs +++ b/binaries/runtime/src/lib.rs @@ -14,7 +14,6 @@ use operator::{run_operator, OperatorEvent, StopReason}; #[cfg(feature = "tracing")] use dora_tracing::set_up_tracing; use std::{ - borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, mem, }; @@ -123,19 +122,17 @@ async fn run( init_done: oneshot::Receiver>, ) -> eyre::Result<()> { #[cfg(feature = "metrics")] - let _started = { + let _meter_provider = { use dora_metrics::init_metrics; - use opentelemetry::global; + use opentelemetry::metrics::MeterProvider as _; use opentelemetry_system_metrics::init_process_observer; - let _started = init_metrics().context("Could not create opentelemetry meter")?; - let meter = global::meter(Cow::Borrowed(Box::leak( - config.node_id.to_string().into_boxed_str(), - ))); - init_process_observer(meter).context("could not initiale system metrics observer")?; - _started + let meter_provider = init_metrics().context("Could not create opentelemetry meter")?; + let meter = meter_provider.meter(config.node_id.to_string()); + let _ = + init_process_observer(meter).context("could not initiale system metrics observer")?; + meter_provider }; - init_done .await .wrap_err("the `init_done` channel was closed unexpectedly")? diff --git a/libraries/extensions/telemetry/metrics/Cargo.toml b/libraries/extensions/telemetry/metrics/Cargo.toml index 53e0ade7..c788e90a 100644 --- a/libraries/extensions/telemetry/metrics/Cargo.toml +++ b/libraries/extensions/telemetry/metrics/Cargo.toml @@ -11,4 +11,5 @@ license.workspace = true [dependencies] opentelemetry = { version = "0.21", features = ["metrics"] } opentelemetry-otlp = { version = "0.14.0", features = ["tonic", "metrics"] } -opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio", "metrics"] } +opentelemetry_sdk = { version = "0.21", features = ["rt-tokio", "metrics"] } + \ No newline at end of file diff --git a/libraries/extensions/telemetry/metrics/src/lib.rs b/libraries/extensions/telemetry/metrics/src/lib.rs index e3c3791c..60e33871 100644 --- a/libraries/extensions/telemetry/metrics/src/lib.rs +++ b/libraries/extensions/telemetry/metrics/src/lib.rs @@ -10,6 +10,8 @@ //! [`sysinfo`]: https://github.com/GuillaumeGomez/sysinfo //! [`opentelemetry-rust`]: https://github.com/open-telemetry/opentelemetry-rust +use std::time::Duration; + use opentelemetry::metrics::{self}; use opentelemetry_sdk::{metrics::MeterProvider, runtime}; @@ -34,5 +36,6 @@ pub fn init_metrics() -> metrics::Result { .tonic() .with_export_config(export_config), ) + .with_period(Duration::from_secs(10)) .build() } From 4ca53258adf76128af61541322a75f8e1dc2241f Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 2 Feb 2024 16:56:29 +0100 Subject: [PATCH 103/104] Move opentelemetry implementation to `dora metrics` --- Cargo.lock | 8 ++++---- binaries/runtime/Cargo.toml | 6 ++---- binaries/runtime/src/lib.rs | 13 ++----------- libraries/extensions/telemetry/metrics/Cargo.toml | 3 ++- libraries/extensions/telemetry/metrics/src/lib.rs | 14 +++++++++++--- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ce8f9ee..d15d7bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1536,8 +1536,10 @@ dependencies = [ name = "dora-metrics" version = "0.3.2" dependencies = [ + "eyre", "opentelemetry 0.21.0", "opentelemetry-otlp", + "opentelemetry-system-metrics", "opentelemetry_sdk 0.21.2", ] @@ -1748,8 +1750,6 @@ dependencies = [ "futures", "futures-concurrency", "libloading", - "opentelemetry 0.21.0", - "opentelemetry-system-metrics", "pyo3", "pythonize", "serde_yaml 0.8.26", @@ -1976,9 +1976,9 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.8" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ "indenter", "once_cell", diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index db221f76..2f6b923d 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -15,8 +15,6 @@ dora-operator-api-types = { workspace = true } dora-core = { workspace = true } dora-tracing = { workspace = true, optional = true } dora-metrics = { workspace = true, optional = true } -opentelemetry = { version = "0.21.0", features = ["metrics"], optional = true } -opentelemetry-system-metrics = { version = "0.1.6", optional = true } eyre = "0.6.8" futures = "0.3.21" futures-concurrency = "7.1.0" @@ -37,6 +35,6 @@ aligned-vec = "0.5.0" [features] default = ["tracing", "metrics"] tracing = ["dora-tracing"] -telemetry = ["tracing", "opentelemetry", "tracing-opentelemetry"] -metrics = ["opentelemetry", "opentelemetry-system-metrics", "dora-metrics"] +telemetry = ["tracing", "tracing-opentelemetry"] +metrics = ["dora-metrics"] python = ["pyo3", "dora-operator-api-python", "pythonize", "arrow/pyarrow"] diff --git a/binaries/runtime/src/lib.rs b/binaries/runtime/src/lib.rs index 81ef6f5a..308d59f1 100644 --- a/binaries/runtime/src/lib.rs +++ b/binaries/runtime/src/lib.rs @@ -5,6 +5,7 @@ use dora_core::{ daemon_messages::{NodeConfig, RuntimeConfig}, descriptor::OperatorConfig, }; +use dora_metrics::init_meter_provider; use dora_node_api::{DoraNode, Event}; use eyre::{bail, Context, Result}; use futures::{Stream, StreamExt}; @@ -122,17 +123,7 @@ async fn run( init_done: oneshot::Receiver>, ) -> eyre::Result<()> { #[cfg(feature = "metrics")] - let _meter_provider = { - use dora_metrics::init_metrics; - use opentelemetry::metrics::MeterProvider as _; - use opentelemetry_system_metrics::init_process_observer; - - let meter_provider = init_metrics().context("Could not create opentelemetry meter")?; - let meter = meter_provider.meter(config.node_id.to_string()); - let _ = - init_process_observer(meter).context("could not initiale system metrics observer")?; - meter_provider - }; + let _meter_provider = init_meter_provider(config.node_id.to_string()); init_done .await .wrap_err("the `init_done` channel was closed unexpectedly")? diff --git a/libraries/extensions/telemetry/metrics/Cargo.toml b/libraries/extensions/telemetry/metrics/Cargo.toml index c788e90a..62b50974 100644 --- a/libraries/extensions/telemetry/metrics/Cargo.toml +++ b/libraries/extensions/telemetry/metrics/Cargo.toml @@ -12,4 +12,5 @@ license.workspace = true opentelemetry = { version = "0.21", features = ["metrics"] } opentelemetry-otlp = { version = "0.14.0", features = ["tonic", "metrics"] } opentelemetry_sdk = { version = "0.21", features = ["rt-tokio", "metrics"] } - \ No newline at end of file +eyre = "0.6.12" +opentelemetry-system-metrics = { version = "0.1.6" } diff --git a/libraries/extensions/telemetry/metrics/src/lib.rs b/libraries/extensions/telemetry/metrics/src/lib.rs index 60e33871..d8f0bfaa 100644 --- a/libraries/extensions/telemetry/metrics/src/lib.rs +++ b/libraries/extensions/telemetry/metrics/src/lib.rs @@ -12,10 +12,11 @@ use std::time::Duration; -use opentelemetry::metrics::{self}; -use opentelemetry_sdk::{metrics::MeterProvider, runtime}; - +use eyre::{Context, Result}; +use opentelemetry::metrics::{self, MeterProvider as _}; use opentelemetry_otlp::{ExportConfig, WithExportConfig}; +use opentelemetry_sdk::{metrics::MeterProvider, runtime}; +use opentelemetry_system_metrics::init_process_observer; /// Init opentelemetry meter /// /// Use the default Opentelemetry exporter with default config @@ -39,3 +40,10 @@ pub fn init_metrics() -> metrics::Result { .with_period(Duration::from_secs(10)) .build() } + +pub fn init_meter_provider(meter_id: String) -> Result { + let meter_provider = init_metrics().context("Could not create opentelemetry meter")?; + let meter = meter_provider.meter(meter_id); + let _ = init_process_observer(meter).context("could not initiale system metrics observer")?; + Ok(meter_provider) +} From fecb58be6085f6d3ece467369c6da327c8e3cd1f Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Tue, 13 Feb 2024 09:36:50 +0100 Subject: [PATCH 104/104] Update `bat` dependency to v0.24 --- Cargo.lock | 109 +++++++++++----------------------------- binaries/cli/Cargo.toml | 2 +- 2 files changed, 29 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b82a08..61301cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,12 +631,11 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bat" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4b13b0233143ae151a66e0135d715b65f631d1028c40502cc88182bcb9f4fa" +checksum = "9dcc9e5637c2330d8eb7b920f2aa5d9e184446c258466f825ea1412c7614cc86" dependencies = [ "ansi_colours", - "atty", "bincode", "bugreport", "bytesize", @@ -644,20 +643,21 @@ dependencies = [ "clircle", "console", "content_inspector", - "dirs 5.0.1", - "encoding", + "encoding_rs", + "etcetera", "flate2", "git2", "globset", "grep-cli", - "nu-ansi-term 0.47.0", + "home", + "nu-ansi-term 0.49.0", "once_cell", "path_abs", "plist", "regex", "semver", "serde", - "serde_yaml 0.8.26", + "serde_yaml 0.9.30", "shell-words", "syntect", "thiserror", @@ -969,9 +969,9 @@ checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "clircle" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68bbd985a63de680ab4d1ad77b6306611a8f961b282c8b5ab513e6de934e396" +checksum = "c8e87cbed5354f17bd8ca8821a097fb62599787fe8f611743fad7ee156a0a600" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1795,70 +1795,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "encoding" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" -dependencies = [ - "encoding-index-japanese", - "encoding-index-korean", - "encoding-index-simpchinese", - "encoding-index-singlebyte", - "encoding-index-tradchinese", -] - -[[package]] -name = "encoding-index-japanese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-korean" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-simpchinese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-singlebyte" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding-index-tradchinese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" -dependencies = [ - "encoding_index_tests", -] - -[[package]] -name = "encoding_index_tests" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" - [[package]] name = "encoding_rs" version = "0.8.33" @@ -1918,6 +1854,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if 1.0.0", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -2283,11 +2230,11 @@ dependencies = [ [[package]] name = "git2" -version = "0.16.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" +checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "libc", "libgit2-sys", "log", @@ -2968,9 +2915,9 @@ checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libgit2-sys" -version = "0.14.2+1.5.1" +version = "0.16.2+1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" dependencies = [ "cc", "libc", @@ -3425,11 +3372,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.47.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df031e117bca634c262e9bd3173776844b6c17a90b3741c9163663b4385af76" +checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" dependencies = [ - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index 0990b9e3..9f8f4e51 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -34,7 +34,7 @@ notify = "5.1.0" ctrlc = "3.2.5" tracing = "0.1.36" dora-tracing = { workspace = true, optional = true } -bat = "0.23.0" +bat = "0.24.0" dora-daemon = { workspace = true } dora-coordinator = { workspace = true } dora-runtime = { workspace = true }