Merge branch 'main' into unify-nodes-and-operators

.github/workflows/ci.yml

@@ -72,6 +72,53 @@ jobs:
         timeout-minutes: 15
         run: cargo run --example cxx-dataflow
 
+  CLI:
+    name: "CLI Test"
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, macos-latest]
+      fail-fast: false
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install libacl-dev (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get install -y libacl1-dev
+
+      - name: "Build cli and binaries"
+        timeout-minutes: 30
+        run: |
+          cargo install --path binaries/coordinator
+          cargo install --path binaries/runtime
+          cargo install --path binaries/cli
+
+      - name: "Start dora-coordinator"
+        run: |
+          dora-coordinator &
+
+      - name: "Test dora `list"
+        run: dora-cli list
+
+      - name: "Test new command"
+        run: |
+          dora-cli new test_project
+
+      - name: "Test start and stop command"
+        timeout-minutes: 30
+        run: |
+          cd test_project
+          cargo build --all
+          UUID=$(dora-cli start dataflow.yml)
+          sleep 10
+          dora-cli stop $UUID
+          cd ..
+
+      - name: "Test dora `destroy"
+        run: dora-cli destroy
+
   examples-remote:
     name: "Examples (Remote)"
     runs-on: ubuntu-latest

Cargo.lock

@@ -274,7 +274,7 @@ checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c"
 
 [[package]]
 name = "benchmark-example-node"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -287,7 +287,7 @@ dependencies = [
 
 [[package]]
 name = "benchmark-example-sink"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -375,9 +375,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.9.1"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
 [[package]]
 name = "byte-tools"
@@ -564,7 +564,7 @@ dependencies = [
 
 [[package]]
 name = "communication-layer-pub-sub"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "eyre",
  "iceoryx-rs",
@@ -575,7 +575,7 @@ dependencies = [
 
 [[package]]
 name = "communication-layer-request-reply"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "eyre",
 ]
@@ -840,41 +840,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "darling"
-version = "0.13.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.13.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim 0.10.0",
- "syn",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.13.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
-dependencies = [
- "darling_core",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "dashmap"
 version = "4.0.2"
@@ -913,6 +878,15 @@ dependencies = [
  "generic-array 0.14.5",
 ]
 
+[[package]]
+name = "dirs"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
+dependencies = [
+ "dirs-sys",
+]
+
 [[package]]
 name = "dirs-next"
 version = "2.0.0"
@@ -923,6 +897,17 @@ dependencies = [
  "dirs-sys-next",
 ]
 
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "dirs-sys-next"
 version = "0.1.2"
@@ -936,7 +921,7 @@ dependencies = [
 
 [[package]]
 name = "dora-cli"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "atty",
  "clap 4.0.3",
@@ -955,7 +940,7 @@ dependencies = [
 
 [[package]]
 name = "dora-coordinator"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "bincode",
  "clap 3.2.20",
@@ -985,12 +970,13 @@ dependencies = [
 
 [[package]]
 name = "dora-core"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-message",
  "eyre",
  "once_cell",
  "serde",
+ "serde-with-expand-env",
  "serde_yaml 0.9.11",
  "tracing",
  "uuid 1.2.1",
@@ -1025,7 +1011,7 @@ dependencies = [
 
 [[package]]
 name = "dora-download"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "eyre",
  "reqwest",
@@ -1051,7 +1037,7 @@ dependencies = [
 
 [[package]]
 name = "dora-message"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "capnp",
  "capnpc",
@@ -1061,7 +1047,7 @@ dependencies = [
 
 [[package]]
 name = "dora-metrics"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "futures",
  "opentelemetry",
@@ -1072,7 +1058,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "bincode",
  "capnp",
@@ -1093,7 +1079,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-c"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -1103,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-cxx"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -1113,7 +1099,7 @@ dependencies = [
 
 [[package]]
 name = "dora-node-api-python"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "dora-operator-api-python",
@@ -1126,7 +1112,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-operator-api-macros",
  "dora-operator-api-types",
@@ -1134,14 +1120,14 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-c"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-operator-api-types",
 ]
 
 [[package]]
 name = "dora-operator-api-cxx"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -1154,7 +1140,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-macros"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-operator-api",
  "dora-operator-api-types",
@@ -1165,7 +1151,7 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-python"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -1176,14 +1162,14 @@ dependencies = [
 
 [[package]]
 name = "dora-operator-api-types"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "safer-ffi",
 ]
 
 [[package]]
 name = "dora-runtime"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-core",
  "dora-download",
@@ -1208,7 +1194,7 @@ dependencies = [
 
 [[package]]
 name = "dora-tracing"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "opentelemetry",
  "opentelemetry-jaeger",
@@ -1741,12 +1727,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -1868,9 +1848,9 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
 
 [[package]]
 name = "jni"
-version = "0.19.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec"
+checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c"
 dependencies = [
  "cesu8",
  "combine",
@@ -2201,64 +2181,12 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "ndk"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0"
-dependencies = [
- "bitflags",
- "jni-sys",
- "ndk-sys",
- "num_enum",
- "raw-window-handle",
- "thiserror",
-]
-
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
 
-[[package]]
-name = "ndk-glue"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0434fabdd2c15e0aab768ca31d5b7b333717f03cf02037d5a0a3ff3c278ed67f"
-dependencies = [
- "libc",
- "log",
- "ndk",
- "ndk-context",
- "ndk-macro",
- "ndk-sys",
- "once_cell",
- "parking_lot",
-]
-
-[[package]]
-name = "ndk-macro"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0df7ac00c4672f9d5aece54ee3347520b7e20f158656c7db2e6de01902eb7a6c"
-dependencies = [
- "darling",
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "ndk-sys"
-version = "0.4.1+23.1.7779620"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cf2aae958bd232cac5069850591667ad422d263686d75b52a065f9badeee5a3"
-dependencies = [
- "jni-sys",
-]
-
 [[package]]
 name = "newline-converter"
 version = "0.2.2"
@@ -2384,27 +2312,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "num_enum"
-version = "0.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9"
-dependencies = [
- "num_enum_derive",
-]
-
-[[package]]
-name = "num_enum_derive"
-version = "0.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce"
-dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "num_threads"
 version = "0.1.5"
@@ -2874,17 +2781,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "proc-macro-crate"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9"
-dependencies = [
- "once_cell",
- "thiserror",
- "toml",
-]
-
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -3321,7 +3217,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-node"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -3332,7 +3228,7 @@ dependencies = [
 
 [[package]]
 name = "rust-dataflow-example-sink"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "dora-node-api",
  "eyre",
@@ -3534,6 +3430,16 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-with-expand-env"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "888d884a3be3a209308d0b66f1918ff18f60e93db837259e53ea7d8dd14e7e98"
+dependencies = [
+ "serde",
+ "shellexpand",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.152"
@@ -3628,7 +3534,7 @@ dependencies = [
 
 [[package]]
 name = "shared-memory-server"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "bincode",
  "eyre",
@@ -3938,9 +3844,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.21.2"
+version = "1.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
+checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb"
 dependencies = [
  "autocfg 1.1.0",
  "bytes",
@@ -3953,7 +3859,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -4026,15 +3932,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "toml"
-version = "0.5.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "tonic"
 version = "0.6.2"
@@ -4524,12 +4421,15 @@ dependencies = [
 
 [[package]]
 name = "webbrowser"
-version = "0.8.0"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01d62aa75495ab67cdc273d0b95cc76bcedfea2ba28338a4cf9b4137949dfac5"
+checksum = "aa61ff77f695a94d9c8558e0bb5c362a8fd1f27c74663770fbc633acbafedbb6"
 dependencies = [
+ "core-foundation",
+ "dirs",
  "jni",
- "ndk-glue",
+ "log",
+ "ndk-context",
  "objc",
  "raw-window-handle",
  "url",
@@ -4650,6 +4550,21 @@ dependencies = [
  "windows_x86_64_msvc 0.36.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.42.1",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.45.0"
@@ -5071,7 +4986,7 @@ dependencies = [
 
 [[package]]
 name = "zenoh-logger"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "zenoh",
  "zenoh-config",

Cargo.toml

@@ -23,12 +23,12 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.2"
+version = "0.1.3"
 
 [workspace.dependencies]
-dora-node-api = { version = "0.1.2", path = "apis/rust/node", default-features = false }
-dora-operator-api = { version = "0.1.2", path = "apis/rust/operator", default-features = false }
-dora-core = { version = "0.1.2", path = "libraries/core" }
+dora-node-api = { version = "0.1.3", path = "apis/rust/node", default-features = false }
+dora-operator-api = { version = "0.1.3", path = "apis/rust/operator", default-features = false }
+dora-core = { version = "0.1.3", path = "libraries/core" }
 
 [package]
 name = "dora-examples"
@@ -39,7 +39,7 @@ license = "Apache-2.0"
 
 [dev-dependencies]
 eyre = "0.6.8"
-tokio = "1.20.1"
+tokio = "1.24.2"
 dora-daemon = { path = "binaries/daemon" }
 dora-core = { path = "libraries/core" }
 dunce = "1.0.2"

Changelog.md

@@ -0,0 +1,31 @@
+# Changelog
+
+## v0.1.3 (2023-01-18)
+
+* Package `DoraStatus` into dora python package: https://github.com/dora-rs/dora/pull/172
+* Force removal of Pyo3 Object to avoid memory leak: https://github.com/dora-rs/dora/pull/168
+* Bump tokio from 1.21.2 to 1.23.1: https://github.com/dora-rs/dora/pull/171
+* Create a changelog file: https://github.com/dora-rs/dora/pull/174 
+
+## v0.1.2 (2022-12-15)
+
+- Fix infinite loop in the coordinator: https://github.com/dora-rs/dora/pull/155
+- Simplify the release process: https://github.com/dora-rs/dora/pull/157
+- Use generic linux distribution: https://github.com/dora-rs/dora/pull/159
+
+## v0.1.1 (2022-12-05)
+
+This release contains fixes for:
+- Python linking using pypi release but also a redesigned python thread model within the runtime to avoid deadlock of the `GIL`. This also fix an issue with `patchelf`.
+- A deployment separation for `ubuntu` as the `20.04` version of `dora` and `22.04` version of dora are non-compatible.
+- A better tagging of api for `dora` Rust API.
+
+## v0.1.0 (2022-11-15)
+
+This is our first release of `dora-rs`!
+
+The current release includes:
+- `dora-cli` which enables creating, starting and stopping dataflow.
+- `dora-coordinator` which is our control plane.
+- `dora-runtime` which is manage the runtime of operators.
+- `custom-nodes` API which enables bridges from different languages.

apis/c++/operator/Cargo.toml

@@ -12,7 +12,7 @@ dora-operator-api = { workspace = true }
 eyre = "0.6.8"
 futures = "0.3.21"
 rand = "0.8.5"
-tokio = { version = "1.20.1", features = ["rt", "macros"] }
+tokio = { version = "1.24.2", features = ["rt", "macros"] }
 
 [build-dependencies]
 cxx-build = "1.0.73"

apis/python/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
-name = "dora-node-api-python"
 version.workspace = true
+name = "dora-node-api-python"
 edition = "2021"
 license = "Apache-2.0"
 

apis/python/node/dora/__init__.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+from .dora import *
+
+
+class DoraStatus(Enum):
+    """Dora status to indicate if operator `on_input` loop
+     should be stopped.
+
+    Args:
+        Enum (u8): Status signaling to dora operator to
+        stop or continue the operator.
+    """
+
+    CONTINUE = 0
+    STOP = 1
+    STOP_ALL = 2

apis/python/node/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["maturin>=0.13.2,<0.14"]
+requires = ["maturin>=0.13.2"]
 build-backend = "maturin"
 
 [project]

apis/rust/node/Cargo.toml

@@ -25,4 +25,4 @@ capnp = "0.14.11"
 bincode = "1.3.3"
 
 [dev-dependencies]
-tokio = { version = "1.17.0", features = ["rt"] }
+tokio = { version = "1.24.2", features = ["rt"] }

binaries/cli/Cargo.toml

@@ -16,7 +16,7 @@ dora-core = { path = "../../libraries/core" }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_yaml = "0.9.11"
 tempfile = "3.3.0"
-webbrowser = "0.8.0"
+webbrowser = "0.8.3"
 serde_json = "1.0.86"
 termcolor = "1.1.3"
 atty = "0.2.14"

binaries/cli/src/template/python/mod.rs

@@ -89,7 +89,7 @@ fn create_dataflow(name: String, path: Option<PathBuf>) -> Result<(), eyre::ErrR
     create_custom_node("node_1".into(), Some(root.join("node_1")))?;
 
     println!(
-        "Created new C dataflow at `{name}` at {}",
+        "Created new yaml dataflow `{name}` at {}",
         Path::new(".").join(root).display()
     );
 

binaries/cli/src/template/python/operator/operator-template.py

@@ -1,11 +1,6 @@
 from typing import Callable
 
-from enum import Enum
-
-
-class DoraStatus(Enum):
-    CONTINUE = 0
-    STOP = 1
+from dora import DoraStatus
 
 
 class Operator:

binaries/cli/src/template/rust/mod.rs

@@ -4,6 +4,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
+const VERSION: &str = env!("CARGO_PKG_VERSION");
 pub fn create(args: crate::CommandNew) -> eyre::Result<()> {
     let crate::CommandNew {
         kind,
@@ -81,7 +82,9 @@ fn create_operator(name: String, path: Option<PathBuf>) -> Result<(), eyre::ErrR
     fs::create_dir(&src)
         .with_context(|| format!("failed to create directory `{}`", src.display()))?;
 
-    let cargo_toml = CARGO_TOML.replace("___name___", &name);
+    let cargo_toml = CARGO_TOML
+        .replace("___name___", &name)
+        .replace("___version___", &VERSION);
     let cargo_toml_path = root.join("Cargo.toml");
     fs::write(&cargo_toml_path, &cargo_toml)
         .with_context(|| format!("failed to write `{}`", cargo_toml_path.display()))?;
@@ -117,7 +120,9 @@ fn create_custom_node(name: String, path: Option<PathBuf>) -> Result<(), eyre::E
     fs::create_dir(&src)
         .with_context(|| format!("failed to create directory `{}`", src.display()))?;
 
-    let cargo_toml = CARGO_TOML.replace("___name___", &name);
+    let cargo_toml = CARGO_TOML
+        .replace("___name___", &name)
+        .replace("___version___", &VERSION);
     let cargo_toml_path = root.join("Cargo.toml");
     fs::write(&cargo_toml_path, &cargo_toml)
         .with_context(|| format!("failed to write `{}`", cargo_toml_path.display()))?;

binaries/cli/src/template/rust/node/Cargo-template.toml

@@ -6,4 +6,4 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-dora-node-api = { git = "https://github.com/dora-rs/dora.git", tag = "v0.1.2" }
+dora-node-api = { git = "https://github.com/dora-rs/dora.git", tag = "v___version___" }

binaries/cli/src/template/rust/operator/Cargo-template.toml

@@ -9,4 +9,4 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [dependencies]
-dora-operator-api = { git = "https://github.com/dora-rs/dora.git", tag = "v0.1.2" }
+dora-operator-api = { git = "https://github.com/dora-rs/dora.git", tag = "v___version___" }

binaries/coordinator/Cargo.toml

@@ -13,7 +13,7 @@ eyre = "0.6.7"
 futures = "0.3.21"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_yaml = "0.8.23"
-tokio = { version = "1.21.2", features = ["full"] }
+tokio = { version = "1.24.2", features = ["full"] }
 tokio-stream = { version = "0.1.8", features = ["io-util", "net"] }
 tokio-util = { version = "0.7.1", features = ["codec"] }
 clap = { version = "3.1.8", features = ["derive"] }

binaries/coordinator/src/run/mod.rs

@@ -13,6 +13,7 @@ use std::{
     path::Path,
 };
 use tokio::net::TcpStream;
+use tracing::warn;
 use uuid::Uuid;
 
 mod runtime;

binaries/runtime/Cargo.toml

@@ -23,7 +23,7 @@ futures = "0.3.21"
 futures-concurrency = "7.1.0"
 libloading = "0.7.3"
 serde_yaml = "0.8.23"
-tokio = { version = "1.17.0", features = ["full"] }
+tokio = { version = "1.24.2", features = ["full"] }
 tokio-stream = "0.1.8"
 # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html
 pyo3 = { version = "0.16", features = ["eyre", "abi3-py37"] }

binaries/runtime/src/operator/python.rs

@@ -8,7 +8,7 @@ use dora_core::{
 use dora_download::download_file;
 use dora_operator_api_python::metadata_to_pydict;
 use dora_operator_api_types::DoraStatus;
-use eyre::{bail, eyre, Context};
+use eyre::{bail, eyre, Context, Result};
 use pyo3::{
     pyclass,
     types::IntoPyDict,
@@ -138,23 +138,35 @@ pub fn run(
                     };
                     metadata.parameters.open_telemetry_context = Cow::Owned(string_cx);
 
-                    let status_enum = Python::with_gil(|py| {
+                    let status = Python::with_gil(|py| -> Result<i32> {
+                        // We need to create a new scoped `GILPool` because the dora-runtime
+                        // is currently started through a `start_runtime` wrapper function,
+                        // which is annotated with `#[pyfunction]`. This attribute creates an
+                        // initial `GILPool` that lasts for the entire lifetime of the `dora-runtime`.
+                        // However, we want the `PyBytes` created below to be freed earlier.
+                        // creating a new scoped `GILPool` tied to this closure, will free `PyBytes`
+                        // at the end of the closure.
+                        // See https://github.com/PyO3/pyo3/pull/2864 and
+                        // https://github.com/PyO3/pyo3/issues/2853 for more details.
+                        let pool = unsafe { py.new_pool() };
+                        let py = pool.python();
                         let input_dict = PyDict::new(py);
 
                         input_dict.set_item("id", input_id.as_str())?;
                         if let Some(data) = data {
-                            input_dict.set_item("data", PyBytes::new(py, &data))?;
+                            let bytes = PyBytes::new(py, &data);
+                            input_dict.set_item("data", bytes)?;
                         }
                         input_dict.set_item("metadata", metadata_to_pydict(&metadata, py))?;
 
-                        operator
+                        let status_enum = operator
                             .call_method1(py, "on_input", (input_dict, send_output.clone()))
-                            .map_err(traceback)
+                            .map_err(traceback)?;
+                        let status_val = Python::with_gil(|py| status_enum.getattr(py, "value"))
+                            .wrap_err("on_input must have enum return value")?;
+                        Python::with_gil(|py| status_val.extract(py))
+                            .wrap_err("on_input has invalid return value")
                     })?;
-                    let status_val = Python::with_gil(|py| status_enum.getattr(py, "value"))
-                        .wrap_err("on_input must have enum return value")?;
-                    let status: i32 = Python::with_gil(|py| status_val.extract(py))
-                        .wrap_err("on_input has invalid return value")?;
                     match status {
                         s if s == DoraStatus::Continue as i32 => {} // ok
                         s if s == DoraStatus::Stop as i32 => break StopReason::ExplicitStop,

examples/python-dataflow/object_detection.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from enum import Enum
 from typing import Callable
 from dora import Node
 

examples/python-dataflow/plot.py

@@ -2,24 +2,17 @@
 # -*- coding: utf-8 -*-
 
 import os
-from enum import Enum
 from typing import Callable
 from dora import Node
+from dora import DoraStatus
 
 import cv2
 import numpy as np
-
 from utils import LABELS
 
 CI = os.environ.get("CI")
 
 font = cv2.FONT_HERSHEY_SIMPLEX
-
-class Status(Enum):
-    CONTINUE = 0
-    STOP = 1
-
-
 class Plotter:
     """
     Plot image and bounding box
@@ -32,7 +25,7 @@ class Plotter:
     def on_input(
         self,
         dora_input: dict,
-    ) -> Status:
+    ) -> DoraStatus:
         """
         Put image and bounding box on cv2 window.
 
@@ -79,12 +72,10 @@ class Plotter:
         if CI != "true":
             cv2.imshow("frame", self.image)
             if cv2.waitKey(1) & 0xFF == ord("q"):
-                return Status.STOP
+                return DoraStatus.STOP
 
-        return Status.CONTINUE
+        return DoraStatus.CONTINUE
 
-    def __del__(self):
-        cv2.destroyAllWindows()
 
 
 plotter = Plotter()
@@ -95,9 +86,9 @@ for event in node:
         case "INPUT":
             status = plotter.on_input(event)
             match status:
-                case Status.CONTINUE:
+                case DoraStatus.CONTINUE:
                     pass
-                case Status.STOP:
+                case DoraStatus.STOP:
                     print("plotter returned stop status")
                     break
         case "STOP":

examples/python-dataflow/requirements.txt

@@ -2,17 +2,20 @@
 # Usage: pip install -r requirements.txt
 
 # Base ----------------------------------------
+gitpython
+ipython  # interactive notebook
 matplotlib>=3.2.2
 numpy>=1.18.5
 opencv-python>=4.1.1
 Pillow>=7.1.2
+psutil  # system resources
 PyYAML>=5.3.1
 requests>=2.23.0
 scipy>=1.4.1
-torch>=1.7.0
+thop>=0.1.1  # FLOPs computation
+torch>=1.7.0  # see https://pytorch.org/get-started/locally (recommended)
 torchvision>=0.8.1
 tqdm>=4.64.0
-protobuf<=3.20.1  # https://github.com/ultralytics/yolov5/issues/8012
 
 # Logging -------------------------------------
 tensorboard>=2.4.1
@@ -35,9 +38,6 @@ seaborn>=0.11.0
 # openvino-dev  # OpenVINO export
 
 # Extras --------------------------------------
-ipython  # interactive notebook
-psutil  # system utilization
-thop>=0.1.1  # FLOPs computation
 # albumentations>=1.0.3
 # pycocotools>=2.0  # COCO mAP
 # roboflow

examples/rust-dataflow/node/Cargo.toml

@@ -10,4 +10,4 @@ dora-node-api = { workspace = true }
 eyre = "0.6.8"
 futures = "0.3.21"
 rand = "0.8.5"
-tokio = { version = "1.20.1", features = ["rt", "macros"] }
+tokio = { version = "1.24.2", features = ["rt", "macros"] }

libraries/core/Cargo.toml

@@ -16,3 +16,4 @@ which = "4.3.0"
 uuid = { version = "1.2.1", features = ["serde"] }
 dora-message = { path = "../message" }
 tracing = "0.1"
+serde-with-expand-env = "1.1.0"

libraries/core/src/descriptor/mod.rs

@@ -4,6 +4,7 @@ use crate::{
 };
 use eyre::{bail, Result};
 use serde::{Deserialize, Serialize};
+use serde_with_expand_env::with_expand_envs;
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap},
     env::consts::EXE_EXTENSION,
@@ -225,8 +226,11 @@ pub struct CustomNode {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum EnvValue {
+    #[serde(deserialize_with = "with_expand_envs")]
     Bool(bool),
+    #[serde(deserialize_with = "with_expand_envs")]
     Integer(u64),
+    #[serde(deserialize_with = "with_expand_envs")]
     String(String),
 }
 

libraries/extensions/download/Cargo.toml

@@ -10,5 +10,5 @@ license = "Apache-2.0"
 eyre = "0.6.8"
 tempfile = "3.3.0"
 reqwest = "0.11.12"
-tokio = { version = "1.17.0" }
+tokio = { version = "1.24.2" }
 tracing = "0.1.36"

libraries/extensions/telemetry/metrics/Cargo.toml

@@ -11,4 +11,4 @@ futures = "0.3.21"
 opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] }
 opentelemetry-otlp = { version = "0.10", features = ["tonic", "metrics"] }
 opentelemetry-system-metrics = "0.1.1"
-tokio = { version = "1.17.0", features = ["full"] }
+tokio = { version = "1.24.2", features = ["full"] }

libraries/extensions/telemetry/tracing/Cargo.toml

@@ -9,4 +9,4 @@ license = "Apache-2.0"
 [dependencies]
 opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] }
 opentelemetry-jaeger = { version = "0.16", features = ["rt-tokio"] }
-tokio = { version = "1.17.0", features = ["full"] }
+tokio = { version = "1.24.2", features = ["full"] }