diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index e95aee8a..74fcd1fb 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -39,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index efe6ff09..7554a5e2 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -39,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 5d873f02..60ecc106 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,3 +1,4 @@ +use dora_core::{get_pip_path, get_python_path, run, set_up_tracing}; use dora_tracing::set_up_tracing; use eyre::{ContextCompat, WrapErr}; use std::path::Path; @@ -11,15 +12,8 @@ async fn main() -> eyre::Result<()> { .wrap_err("failed to set working dir")?; run( - &[ - get_python_path() - .context("Could not get python binary")? - .to_str() - .context("Could not convert python path to string")?, - "-m", - "venv", - ".env", - ], + get_python_path().context("Could not get python binary")?, + &["-m", "venv", "../.env"], None, ) .await @@ -27,9 +21,7 @@ async fn main() -> eyre::Result<()> { let venv = &root.join("examples").join(".env"); std::env::set_var( "VIRTUAL_ENV", - venv.to_str() - .context("venv path not valid unicode")? - .to_owned(), + venv.to_str().context("venv path not valid unicode")?, ); let orig_path = std::env::var("PATH")?; let venv_bin = venv.join("bin"); @@ -41,15 +33,24 @@ async fn main() -> eyre::Result<()> { ), ); - run(&["pip", "install", "--upgrade", "pip"], None) - .await - .context("failed to install pip")?; - run(&["pip", "install", "-r", "requirements.txt"], None) - .await - .context("pip install failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "--upgrade", "pip"], + None, + ) + .await + .context("failed to install pip")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "-r", "requirements.txt"], + None, + ) + .await + .context("pip install failed")?; run( - &["maturin", "develop"], + "maturin", + &["develop"], Some(&root.join("apis").join("python").join("node")), ) .await diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index 3be48e23..c5961e1d 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -19,5 +19,5 @@ uuid = { version = "1.2.1", features = ["serde"] } dora-message = { workspace = true } tracing = "0.1" serde-with-expand-env = "1.1.0" -tokio = { version = "1.24.1", features = ["fs"] } +tokio = { version = "1.24.1", features = ["fs", "process"] } aligned-vec = { version = "0.5.0", features = ["serde"] } diff --git a/libraries/core/src/lib.rs b/libraries/core/src/lib.rs index 9b6d81b8..e0145d2b 100644 --- a/libraries/core/src/lib.rs +++ b/libraries/core/src/lib.rs @@ -1,6 +1,7 @@ use eyre::{bail, eyre, Context}; use std::{ env::consts::{DLL_PREFIX, DLL_SUFFIX}, + ffi::OsStr, path::Path, }; @@ -33,12 +34,41 @@ pub fn adjust_shared_library_path(path: &Path) -> Result Result { let python = match which::which("python3") { - Ok(python) => python, - Err(_) => which::which("python") - .context("failed to find `python` or `python3` in dora-daemon path. Make sure that python is available for the daemon.")?, - }; + Ok(python) => python, + Err(_) => which::which("python") + .context("failed to find `python` or `python3`. Make sure that python is available.")?, + }; Ok(python) } + +// Search for pip binary. +// First search for `pip3` as for ubuntu <20, `pip` can resolves to `python2,7 -m pip` +// Then search for `pip`, this will resolve for windows to python3 -m pip. +pub fn get_pip_path() -> Result { + let python = match which::which("pip3") { + Ok(python) => python, + Err(_) => which::which("pip") + .context("failed to find `pip3` or `pip`. Make sure that python is available.")?, + }; + Ok(python) +} + +// Helper function to run a program +pub async fn run(program: S, args: &[&str], pwd: Option<&Path>) -> eyre::Result<()> +where + S: AsRef, +{ + let mut run = tokio::process::Command::new(program); + run.args(args); + + if let Some(pwd) = pwd { + run.current_dir(pwd); + } + if !run.status().await?.success() { + eyre::bail!("failed to run {args:?}"); + }; + Ok(()) +}