diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b5d13dd..e1e40f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,53 @@ jobs: timeout-minutes: 15 run: cargo run --example cxx-dataflow + CLI: + name: "CLI Test" + strategy: + matrix: + platform: [ubuntu-latest, macos-latest] + fail-fast: false + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v3 + + - name: Install libacl-dev (Linux) + if: runner.os == 'Linux' + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get install -y libacl1-dev + + - name: "Build cli and binaries" + timeout-minutes: 30 + run: | + cargo install --path binaries/coordinator + cargo install --path binaries/runtime + cargo install --path binaries/cli + + - name: "Start dora-coordinator" + run: | + dora-coordinator & + + - name: "Test dora `list" + run: dora-cli list + + - name: "Test new command" + run: | + dora-cli new test_project + + - name: "Test start and stop command" + timeout-minutes: 30 + run: | + cd test_project + cargo build --all + UUID=$(dora-cli start dataflow.yml) + sleep 10 + dora-cli stop $UUID + cd .. + + - name: "Test dora `destroy" + run: dora-cli destroy + examples-remote: name: "Examples (Remote)" runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 5ec3dcd8..f8a15eb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,7 +274,7 @@ checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c" [[package]] name = "benchmark-example-node" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -287,7 +287,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.9.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byte-tools" @@ -564,7 +564,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.1.2" +version = "0.1.3" dependencies = [ "eyre", "iceoryx-rs", @@ -575,7 +575,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.1.2" +version = "0.1.3" dependencies = [ "eyre", ] @@ -840,41 +840,6 @@ dependencies = [ "syn", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core", - "quote", - "syn", -] - [[package]] name = "dashmap" version = "4.0.2" @@ -913,6 +878,15 @@ dependencies = [ "generic-array 0.14.5", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -923,6 +897,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -936,7 +921,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.1.2" +version = "0.1.3" dependencies = [ "atty", "clap 4.0.3", @@ -955,7 +940,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.1.2" +version = "0.1.3" dependencies = [ "bincode", "clap 3.2.20", @@ -985,12 +970,13 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-message", "eyre", "once_cell", "serde", + "serde-with-expand-env", "serde_yaml 0.9.11", "tracing", "uuid 1.2.1", @@ -1025,7 +1011,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.1.2" +version = "0.1.3" dependencies = [ "eyre", "reqwest", @@ -1051,7 +1037,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.1.2" +version = "0.1.3" dependencies = [ "capnp", "capnpc", @@ -1061,7 +1047,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.1.2" +version = "0.1.3" dependencies = [ "futures", "opentelemetry", @@ -1072,7 +1058,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.1.2" +version = "0.1.3" dependencies = [ "bincode", "capnp", @@ -1093,7 +1079,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -1103,7 +1089,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.1.2" +version = "0.1.3" dependencies = [ "cxx", "cxx-build", @@ -1113,7 +1099,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "dora-operator-api-python", @@ -1126,7 +1112,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-operator-api-macros", "dora-operator-api-types", @@ -1134,14 +1120,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.1.2" +version = "0.1.3" dependencies = [ "cxx", "cxx-build", @@ -1154,7 +1140,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-operator-api", "dora-operator-api-types", @@ -1165,7 +1151,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -1176,14 +1162,14 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.1.2" +version = "0.1.3" dependencies = [ "safer-ffi", ] [[package]] name = "dora-runtime" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-core", "dora-download", @@ -1208,7 +1194,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.1.2" +version = "0.1.3" dependencies = [ "opentelemetry", "opentelemetry-jaeger", @@ -1741,12 +1727,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "0.2.3" @@ -1868,9 +1848,9 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "jni" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" dependencies = [ "cesu8", "combine", @@ -2201,64 +2181,12 @@ dependencies = [ "tempfile", ] -[[package]] -name = "ndk" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" -dependencies = [ - "bitflags", - "jni-sys", - "ndk-sys", - "num_enum", - "raw-window-handle", - "thiserror", -] - [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" -[[package]] -name = "ndk-glue" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0434fabdd2c15e0aab768ca31d5b7b333717f03cf02037d5a0a3ff3c278ed67f" -dependencies = [ - "libc", - "log", - "ndk", - "ndk-context", - "ndk-macro", - "ndk-sys", - "once_cell", - "parking_lot", -] - -[[package]] -name = "ndk-macro" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df7ac00c4672f9d5aece54ee3347520b7e20f158656c7db2e6de01902eb7a6c" -dependencies = [ - "darling", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "ndk-sys" -version = "0.4.1+23.1.7779620" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf2aae958bd232cac5069850591667ad422d263686d75b52a065f9badeee5a3" -dependencies = [ - "jni-sys", -] - [[package]] name = "newline-converter" version = "0.2.2" @@ -2384,27 +2312,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_enum" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "num_threads" version = "0.1.5" @@ -2874,17 +2781,6 @@ dependencies = [ "syn", ] -[[package]] -name = "proc-macro-crate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" -dependencies = [ - "once_cell", - "thiserror", - "toml", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3321,7 +3217,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -3332,7 +3228,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-sink" -version = "0.1.2" +version = "0.1.3" dependencies = [ "dora-node-api", "eyre", @@ -3534,6 +3430,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-with-expand-env" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "888d884a3be3a209308d0b66f1918ff18f60e93db837259e53ea7d8dd14e7e98" +dependencies = [ + "serde", + "shellexpand", +] + [[package]] name = "serde_derive" version = "1.0.152" @@ -3628,7 +3534,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.1.2" +version = "0.1.3" dependencies = [ "bincode", "eyre", @@ -3938,9 +3844,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg 1.1.0", "bytes", @@ -3953,7 +3859,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -4026,15 +3932,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" -dependencies = [ - "serde", -] - [[package]] name = "tonic" version = "0.6.2" @@ -4524,12 +4421,15 @@ dependencies = [ [[package]] name = "webbrowser" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d62aa75495ab67cdc273d0b95cc76bcedfea2ba28338a4cf9b4137949dfac5" +checksum = "aa61ff77f695a94d9c8558e0bb5c362a8fd1f27c74663770fbc633acbafedbb6" dependencies = [ + "core-foundation", + "dirs", "jni", - "ndk-glue", + "log", + "ndk-context", "objc", "raw-window-handle", "url", @@ -4650,6 +4550,21 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.1", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -5071,7 +4986,7 @@ dependencies = [ [[package]] name = "zenoh-logger" -version = "0.1.2" +version = "0.1.3" dependencies = [ "zenoh", "zenoh-config", diff --git a/Cargo.toml b/Cargo.toml index 30148c22..84fbb1e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,12 +23,12 @@ members = [ ] [workspace.package] -version = "0.1.2" +version = "0.1.3" [workspace.dependencies] -dora-node-api = { version = "0.1.2", path = "apis/rust/node", default-features = false } -dora-operator-api = { version = "0.1.2", path = "apis/rust/operator", default-features = false } -dora-core = { version = "0.1.2", path = "libraries/core" } +dora-node-api = { version = "0.1.3", path = "apis/rust/node", default-features = false } +dora-operator-api = { version = "0.1.3", path = "apis/rust/operator", default-features = false } +dora-core = { version = "0.1.3", path = "libraries/core" } [package] name = "dora-examples" @@ -39,7 +39,7 @@ license = "Apache-2.0" [dev-dependencies] eyre = "0.6.8" -tokio = "1.20.1" +tokio = "1.24.2" dora-daemon = { path = "binaries/daemon" } dora-core = { path = "libraries/core" } dunce = "1.0.2" diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 00000000..e457aebc --- /dev/null +++ b/Changelog.md @@ -0,0 +1,31 @@ +# Changelog + +## v0.1.3 (2023-01-18) + +* Package `DoraStatus` into dora python package: https://github.com/dora-rs/dora/pull/172 +* Force removal of Pyo3 Object to avoid memory leak: https://github.com/dora-rs/dora/pull/168 +* Bump tokio from 1.21.2 to 1.23.1: https://github.com/dora-rs/dora/pull/171 +* Create a changelog file: https://github.com/dora-rs/dora/pull/174 + +## v0.1.2 (2022-12-15) + +- Fix infinite loop in the coordinator: https://github.com/dora-rs/dora/pull/155 +- Simplify the release process: https://github.com/dora-rs/dora/pull/157 +- Use generic linux distribution: https://github.com/dora-rs/dora/pull/159 + +## v0.1.1 (2022-12-05) + +This release contains fixes for: +- Python linking using pypi release but also a redesigned python thread model within the runtime to avoid deadlock of the `GIL`. This also fix an issue with `patchelf`. +- A deployment separation for `ubuntu` as the `20.04` version of `dora` and `22.04` version of dora are non-compatible. +- A better tagging of api for `dora` Rust API. + +## v0.1.0 (2022-11-15) + +This is our first release of `dora-rs`! + +The current release includes: +- `dora-cli` which enables creating, starting and stopping dataflow. +- `dora-coordinator` which is our control plane. +- `dora-runtime` which is manage the runtime of operators. +- `custom-nodes` API which enables bridges from different languages. diff --git a/apis/c++/operator/Cargo.toml b/apis/c++/operator/Cargo.toml index 6e3a4a82..7b8bf06e 100644 --- a/apis/c++/operator/Cargo.toml +++ b/apis/c++/operator/Cargo.toml @@ -12,7 +12,7 @@ dora-operator-api = { workspace = true } eyre = "0.6.8" futures = "0.3.21" rand = "0.8.5" -tokio = { version = "1.20.1", features = ["rt", "macros"] } +tokio = { version = "1.24.2", features = ["rt", "macros"] } [build-dependencies] cxx-build = "1.0.73" diff --git a/apis/python/node/Cargo.toml b/apis/python/node/Cargo.toml index fdffc278..386dbae7 100644 --- a/apis/python/node/Cargo.toml +++ b/apis/python/node/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "dora-node-api-python" version.workspace = true +name = "dora-node-api-python" edition = "2021" license = "Apache-2.0" diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py new file mode 100644 index 00000000..2893b7ee --- /dev/null +++ b/apis/python/node/dora/__init__.py @@ -0,0 +1,17 @@ +from enum import Enum + +from .dora import * + + +class DoraStatus(Enum): + """Dora status to indicate if operator `on_input` loop + should be stopped. + + Args: + Enum (u8): Status signaling to dora operator to + stop or continue the operator. + """ + + CONTINUE = 0 + STOP = 1 + STOP_ALL = 2 diff --git a/apis/python/node/pyproject.toml b/apis/python/node/pyproject.toml index 878e0b81..f23f5f3d 100644 --- a/apis/python/node/pyproject.toml +++ b/apis/python/node/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13.2,<0.14"] +requires = ["maturin>=0.13.2"] build-backend = "maturin" [project] diff --git a/apis/rust/node/Cargo.toml b/apis/rust/node/Cargo.toml index 56cad481..9d94c2a5 100644 --- a/apis/rust/node/Cargo.toml +++ b/apis/rust/node/Cargo.toml @@ -25,4 +25,4 @@ capnp = "0.14.11" bincode = "1.3.3" [dev-dependencies] -tokio = { version = "1.17.0", features = ["rt"] } +tokio = { version = "1.24.2", features = ["rt"] } diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index efc458e2..5a57abdc 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -16,7 +16,7 @@ dora-core = { path = "../../libraries/core" } serde = { version = "1.0.136", features = ["derive"] } serde_yaml = "0.9.11" tempfile = "3.3.0" -webbrowser = "0.8.0" +webbrowser = "0.8.3" serde_json = "1.0.86" termcolor = "1.1.3" atty = "0.2.14" diff --git a/binaries/cli/src/template/python/mod.rs b/binaries/cli/src/template/python/mod.rs index fb742eef..d9128b46 100644 --- a/binaries/cli/src/template/python/mod.rs +++ b/binaries/cli/src/template/python/mod.rs @@ -89,7 +89,7 @@ fn create_dataflow(name: String, path: Option) -> Result<(), eyre::ErrR create_custom_node("node_1".into(), Some(root.join("node_1")))?; println!( - "Created new C dataflow at `{name}` at {}", + "Created new yaml dataflow `{name}` at {}", Path::new(".").join(root).display() ); diff --git a/binaries/cli/src/template/python/operator/operator-template.py b/binaries/cli/src/template/python/operator/operator-template.py index 36baecef..f503c32a 100644 --- a/binaries/cli/src/template/python/operator/operator-template.py +++ b/binaries/cli/src/template/python/operator/operator-template.py @@ -1,11 +1,6 @@ from typing import Callable -from enum import Enum - - -class DoraStatus(Enum): - CONTINUE = 0 - STOP = 1 +from dora import DoraStatus class Operator: diff --git a/binaries/cli/src/template/rust/mod.rs b/binaries/cli/src/template/rust/mod.rs index b74f0437..c5dc5ab9 100644 --- a/binaries/cli/src/template/rust/mod.rs +++ b/binaries/cli/src/template/rust/mod.rs @@ -4,6 +4,7 @@ use std::{ path::{Path, PathBuf}, }; +const VERSION: &str = env!("CARGO_PKG_VERSION"); pub fn create(args: crate::CommandNew) -> eyre::Result<()> { let crate::CommandNew { kind, @@ -81,7 +82,9 @@ fn create_operator(name: String, path: Option) -> Result<(), eyre::ErrR fs::create_dir(&src) .with_context(|| format!("failed to create directory `{}`", src.display()))?; - let cargo_toml = CARGO_TOML.replace("___name___", &name); + let cargo_toml = CARGO_TOML + .replace("___name___", &name) + .replace("___version___", &VERSION); let cargo_toml_path = root.join("Cargo.toml"); fs::write(&cargo_toml_path, &cargo_toml) .with_context(|| format!("failed to write `{}`", cargo_toml_path.display()))?; @@ -117,7 +120,9 @@ fn create_custom_node(name: String, path: Option) -> Result<(), eyre::E fs::create_dir(&src) .with_context(|| format!("failed to create directory `{}`", src.display()))?; - let cargo_toml = CARGO_TOML.replace("___name___", &name); + let cargo_toml = CARGO_TOML + .replace("___name___", &name) + .replace("___version___", &VERSION); let cargo_toml_path = root.join("Cargo.toml"); fs::write(&cargo_toml_path, &cargo_toml) .with_context(|| format!("failed to write `{}`", cargo_toml_path.display()))?; diff --git a/binaries/cli/src/template/rust/node/Cargo-template.toml b/binaries/cli/src/template/rust/node/Cargo-template.toml index 0052894b..2d609a7e 100644 --- a/binaries/cli/src/template/rust/node/Cargo-template.toml +++ b/binaries/cli/src/template/rust/node/Cargo-template.toml @@ -6,4 +6,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -dora-node-api = { git = "https://github.com/dora-rs/dora.git", tag = "v0.1.2" } +dora-node-api = { git = "https://github.com/dora-rs/dora.git", tag = "v___version___" } diff --git a/binaries/cli/src/template/rust/operator/Cargo-template.toml b/binaries/cli/src/template/rust/operator/Cargo-template.toml index fe920b45..0ee039ca 100644 --- a/binaries/cli/src/template/rust/operator/Cargo-template.toml +++ b/binaries/cli/src/template/rust/operator/Cargo-template.toml @@ -9,4 +9,4 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -dora-operator-api = { git = "https://github.com/dora-rs/dora.git", tag = "v0.1.2" } +dora-operator-api = { git = "https://github.com/dora-rs/dora.git", tag = "v___version___" } diff --git a/binaries/coordinator/Cargo.toml b/binaries/coordinator/Cargo.toml index c75bc79b..d304a7e5 100644 --- a/binaries/coordinator/Cargo.toml +++ b/binaries/coordinator/Cargo.toml @@ -13,7 +13,7 @@ eyre = "0.6.7" futures = "0.3.21" serde = { version = "1.0.136", features = ["derive"] } serde_yaml = "0.8.23" -tokio = { version = "1.21.2", features = ["full"] } +tokio = { version = "1.24.2", features = ["full"] } tokio-stream = { version = "0.1.8", features = ["io-util", "net"] } tokio-util = { version = "0.7.1", features = ["codec"] } clap = { version = "3.1.8", features = ["derive"] } diff --git a/binaries/coordinator/src/run/mod.rs b/binaries/coordinator/src/run/mod.rs index b0e331dc..454cb5c8 100644 --- a/binaries/coordinator/src/run/mod.rs +++ b/binaries/coordinator/src/run/mod.rs @@ -13,6 +13,7 @@ use std::{ path::Path, }; use tokio::net::TcpStream; +use tracing::warn; use uuid::Uuid; mod runtime; diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index 3f358d90..2f972c7d 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -23,7 +23,7 @@ futures = "0.3.21" futures-concurrency = "7.1.0" libloading = "0.7.3" serde_yaml = "0.8.23" -tokio = { version = "1.17.0", features = ["full"] } +tokio = { version = "1.24.2", features = ["full"] } tokio-stream = "0.1.8" # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html pyo3 = { version = "0.16", features = ["eyre", "abi3-py37"] } diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index c7b6f595..be749d1e 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -8,7 +8,7 @@ use dora_core::{ use dora_download::download_file; use dora_operator_api_python::metadata_to_pydict; use dora_operator_api_types::DoraStatus; -use eyre::{bail, eyre, Context}; +use eyre::{bail, eyre, Context, Result}; use pyo3::{ pyclass, types::IntoPyDict, @@ -138,23 +138,35 @@ pub fn run( }; metadata.parameters.open_telemetry_context = Cow::Owned(string_cx); - let status_enum = Python::with_gil(|py| { + let status = Python::with_gil(|py| -> Result { + // We need to create a new scoped `GILPool` because the dora-runtime + // is currently started through a `start_runtime` wrapper function, + // which is annotated with `#[pyfunction]`. This attribute creates an + // initial `GILPool` that lasts for the entire lifetime of the `dora-runtime`. + // However, we want the `PyBytes` created below to be freed earlier. + // creating a new scoped `GILPool` tied to this closure, will free `PyBytes` + // at the end of the closure. + // See https://github.com/PyO3/pyo3/pull/2864 and + // https://github.com/PyO3/pyo3/issues/2853 for more details. + let pool = unsafe { py.new_pool() }; + let py = pool.python(); let input_dict = PyDict::new(py); input_dict.set_item("id", input_id.as_str())?; if let Some(data) = data { - input_dict.set_item("data", PyBytes::new(py, &data))?; + let bytes = PyBytes::new(py, &data); + input_dict.set_item("data", bytes)?; } input_dict.set_item("metadata", metadata_to_pydict(&metadata, py))?; - operator + let status_enum = operator .call_method1(py, "on_input", (input_dict, send_output.clone())) - .map_err(traceback) + .map_err(traceback)?; + let status_val = Python::with_gil(|py| status_enum.getattr(py, "value")) + .wrap_err("on_input must have enum return value")?; + Python::with_gil(|py| status_val.extract(py)) + .wrap_err("on_input has invalid return value") })?; - let status_val = Python::with_gil(|py| status_enum.getattr(py, "value")) - .wrap_err("on_input must have enum return value")?; - let status: i32 = Python::with_gil(|py| status_val.extract(py)) - .wrap_err("on_input has invalid return value")?; match status { s if s == DoraStatus::Continue as i32 => {} // ok s if s == DoraStatus::Stop as i32 => break StopReason::ExplicitStop, diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py index 3f07a18c..07aeabd2 100755 --- a/examples/python-dataflow/object_detection.py +++ b/examples/python-dataflow/object_detection.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from enum import Enum from typing import Callable from dora import Node diff --git a/examples/python-dataflow/plot.py b/examples/python-dataflow/plot.py index e3723d94..b27e1537 100755 --- a/examples/python-dataflow/plot.py +++ b/examples/python-dataflow/plot.py @@ -2,24 +2,17 @@ # -*- coding: utf-8 -*- import os -from enum import Enum from typing import Callable from dora import Node +from dora import DoraStatus import cv2 import numpy as np - from utils import LABELS CI = os.environ.get("CI") font = cv2.FONT_HERSHEY_SIMPLEX - -class Status(Enum): - CONTINUE = 0 - STOP = 1 - - class Plotter: """ Plot image and bounding box @@ -32,7 +25,7 @@ class Plotter: def on_input( self, dora_input: dict, - ) -> Status: + ) -> DoraStatus: """ Put image and bounding box on cv2 window. @@ -79,12 +72,10 @@ class Plotter: if CI != "true": cv2.imshow("frame", self.image) if cv2.waitKey(1) & 0xFF == ord("q"): - return Status.STOP + return DoraStatus.STOP - return Status.CONTINUE + return DoraStatus.CONTINUE - def __del__(self): - cv2.destroyAllWindows() plotter = Plotter() @@ -95,9 +86,9 @@ for event in node: case "INPUT": status = plotter.on_input(event) match status: - case Status.CONTINUE: + case DoraStatus.CONTINUE: pass - case Status.STOP: + case DoraStatus.STOP: print("plotter returned stop status") break case "STOP": diff --git a/examples/python-dataflow/requirements.txt b/examples/python-dataflow/requirements.txt index 55f71178..3245eada 100644 --- a/examples/python-dataflow/requirements.txt +++ b/examples/python-dataflow/requirements.txt @@ -2,17 +2,20 @@ # Usage: pip install -r requirements.txt # Base ---------------------------------------- +gitpython +ipython # interactive notebook matplotlib>=3.2.2 numpy>=1.18.5 opencv-python>=4.1.1 Pillow>=7.1.2 +psutil # system resources PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 -torch>=1.7.0 +thop>=0.1.1 # FLOPs computation +torch>=1.7.0 # see https://pytorch.org/get-started/locally (recommended) torchvision>=0.8.1 tqdm>=4.64.0 -protobuf<=3.20.1 # https://github.com/ultralytics/yolov5/issues/8012 # Logging ------------------------------------- tensorboard>=2.4.1 @@ -35,9 +38,6 @@ seaborn>=0.11.0 # openvino-dev # OpenVINO export # Extras -------------------------------------- -ipython # interactive notebook -psutil # system utilization -thop>=0.1.1 # FLOPs computation # albumentations>=1.0.3 # pycocotools>=2.0 # COCO mAP # roboflow diff --git a/examples/rust-dataflow/node/Cargo.toml b/examples/rust-dataflow/node/Cargo.toml index e9db92d9..9cc86c56 100644 --- a/examples/rust-dataflow/node/Cargo.toml +++ b/examples/rust-dataflow/node/Cargo.toml @@ -10,4 +10,4 @@ dora-node-api = { workspace = true } eyre = "0.6.8" futures = "0.3.21" rand = "0.8.5" -tokio = { version = "1.20.1", features = ["rt", "macros"] } +tokio = { version = "1.24.2", features = ["rt", "macros"] } diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index b4651fec..258cc0b5 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -16,3 +16,4 @@ which = "4.3.0" uuid = { version = "1.2.1", features = ["serde"] } dora-message = { path = "../message" } tracing = "0.1" +serde-with-expand-env = "1.1.0" diff --git a/libraries/core/src/descriptor/mod.rs b/libraries/core/src/descriptor/mod.rs index 41cbde15..8dfdc689 100644 --- a/libraries/core/src/descriptor/mod.rs +++ b/libraries/core/src/descriptor/mod.rs @@ -4,6 +4,7 @@ use crate::{ }; use eyre::{bail, Result}; use serde::{Deserialize, Serialize}; +use serde_with_expand_env::with_expand_envs; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, env::consts::EXE_EXTENSION, @@ -225,8 +226,11 @@ pub struct CustomNode { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] pub enum EnvValue { + #[serde(deserialize_with = "with_expand_envs")] Bool(bool), + #[serde(deserialize_with = "with_expand_envs")] Integer(u64), + #[serde(deserialize_with = "with_expand_envs")] String(String), } diff --git a/libraries/extensions/download/Cargo.toml b/libraries/extensions/download/Cargo.toml index 6155a6e4..f2fa2d4f 100644 --- a/libraries/extensions/download/Cargo.toml +++ b/libraries/extensions/download/Cargo.toml @@ -10,5 +10,5 @@ license = "Apache-2.0" eyre = "0.6.8" tempfile = "3.3.0" reqwest = "0.11.12" -tokio = { version = "1.17.0" } +tokio = { version = "1.24.2" } tracing = "0.1.36" diff --git a/libraries/extensions/telemetry/metrics/Cargo.toml b/libraries/extensions/telemetry/metrics/Cargo.toml index 57d31315..5b904cbf 100644 --- a/libraries/extensions/telemetry/metrics/Cargo.toml +++ b/libraries/extensions/telemetry/metrics/Cargo.toml @@ -11,4 +11,4 @@ futures = "0.3.21" opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] } opentelemetry-otlp = { version = "0.10", features = ["tonic", "metrics"] } opentelemetry-system-metrics = "0.1.1" -tokio = { version = "1.17.0", features = ["full"] } +tokio = { version = "1.24.2", features = ["full"] } diff --git a/libraries/extensions/telemetry/tracing/Cargo.toml b/libraries/extensions/telemetry/tracing/Cargo.toml index 2dd5ae4e..284dab7f 100644 --- a/libraries/extensions/telemetry/tracing/Cargo.toml +++ b/libraries/extensions/telemetry/tracing/Cargo.toml @@ -9,4 +9,4 @@ license = "Apache-2.0" [dependencies] opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] } opentelemetry-jaeger = { version = "0.16", features = ["rt-tokio"] } -tokio = { version = "1.17.0", features = ["full"] } +tokio = { version = "1.24.2", features = ["full"] }