From 93b7da2b74fc444cc65717acddda209a76fc285c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 4 May 2023 14:11:12 +0800 Subject: [PATCH 1/2] Remove pyo3 in runtime and daemon as it generates `libpython` dependencies Previous implementation of the runtime would link `libpython` even though it might not be required. This commit put pyo3 dependency under a feature flag that is only active if the runtime is called from python for a python operator. This fix: - https://github.com/dora-rs/dora/actions/runs/4879188973/jobs/8705515472 - https://github.com/dora-rs/dora/issues/280#issuecomment-1534058207 - and https://github.com/dora-rs/dora-drives/issues/58#event-9162532660 --- apis/python/node/Cargo.toml | 2 +- binaries/runtime/Cargo.toml | 3 ++- binaries/runtime/src/operator/mod.rs | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apis/python/node/Cargo.toml b/apis/python/node/Cargo.toml index 3a971c98..8e92a8c9 100644 --- a/apis/python/node/Cargo.toml +++ b/apis/python/node/Cargo.toml @@ -20,7 +20,7 @@ pyo3 = { version = "0.18", features = ["eyre", "abi3-py37"] } eyre = "0.6" serde_yaml = "0.8.23" flume = "0.10.14" -dora-runtime = { workspace = true, features = ["tracing"] } +dora-runtime = { workspace = true, features = ["tracing", "python"] } arrow = { version = "35.0.0", features = ["pyarrow"] } [lib] diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index 46dea641..fef64c3b 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -28,7 +28,7 @@ serde_yaml = "0.8.23" tokio = { version = "1.24.2", features = ["full"] } tokio-stream = "0.1.8" # pyo3-abi3 flag allow simpler linking. See: https://pyo3.rs/v0.13.2/building_and_distribution.html -pyo3 = { version = "0.18", features = ["eyre", "abi3-py37"] } +pyo3 = { version = "0.18", features = ["eyre", "abi3-py37"], optional = true } tracing = "0.1.36" tracing-subscriber = "0.3.15" dora-download = { workspace = true } @@ -41,3 +41,4 @@ default = ["tracing"] tracing = ["dora-tracing"] telemetry = ["tracing", "opentelemetry", "tracing-opentelemetry"] metrics = ["opentelemetry", "opentelemetry-system-metrics", "dora-metrics"] +python = ["pyo3"] diff --git a/binaries/runtime/src/operator/mod.rs b/binaries/runtime/src/operator/mod.rs index 2fa06c73..b538ad73 100644 --- a/binaries/runtime/src/operator/mod.rs +++ b/binaries/runtime/src/operator/mod.rs @@ -9,6 +9,7 @@ use std::any::Any; use tokio::sync::{mpsc::Sender, oneshot}; pub mod channel; +#[cfg(feature = "python")] mod python; mod shared_lib; @@ -36,7 +37,9 @@ pub fn run_operator( ) })?; } + #[allow(unused_variables)] OperatorSource::Python(source) => { + #[cfg(feature = "python")] python::run( node_id, &operator_definition.id, @@ -51,6 +54,10 @@ pub fn run_operator( operator_definition.id ) })?; + #[cfg(not(feature = "python"))] + tracing::error!( + "Dora runtime tried spawning Python Operator outside of python environment." + ); } OperatorSource::Wasm(_) => { tracing::error!("WASM operators are not supported yet"); From fc0e12fb3b2517831120af55f4c394f5e8d66e41 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 4 May 2023 17:13:12 +0800 Subject: [PATCH 2/2] Remove `api-python` for isolating daemon from python --- binaries/runtime/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/binaries/runtime/Cargo.toml b/binaries/runtime/Cargo.toml index fef64c3b..bdc1976a 100644 --- a/binaries/runtime/Cargo.toml +++ b/binaries/runtime/Cargo.toml @@ -10,7 +10,7 @@ license.workspace = true [dependencies] dora-node-api = { workspace = true, default-features = false } -dora-operator-api-python = { workspace = true } +dora-operator-api-python = { workspace = true, optional = true } dora-operator-api-types = { workspace = true } dora-core = { workspace = true } dora-tracing = { workspace = true, optional = true } @@ -41,4 +41,4 @@ default = ["tracing"] tracing = ["dora-tracing"] telemetry = ["tracing", "opentelemetry", "tracing-opentelemetry"] metrics = ["opentelemetry", "opentelemetry-system-metrics", "dora-metrics"] -python = ["pyo3"] +python = ["pyo3", "dora-operator-api-python"]