From e7cb02311fd5de6fa7eac7bcee29f068693581fe Mon Sep 17 00:00:00 2001 From: haixuantao Date: Fri, 21 Feb 2025 18:42:55 +0100 Subject: [PATCH] Fix venv in operator and ros2 dataflow --- examples/python-operator-dataflow/run.rs | 101 ++++++------------ examples/python-ros2-dataflow/README.md | 11 ++ .../python-ros2-dataflow/requirements.txt | 2 - examples/python-ros2-dataflow/run.rs | 92 ++++++---------- 4 files changed, 75 insertions(+), 131 deletions(-) delete mode 100644 examples/python-ros2-dataflow/requirements.txt diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index d6534cf6..d523d5a4 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,98 +1,63 @@ -use dora_core::{get_pip_path, get_python_path, run}; +use dora_core::{get_uv_path, run}; use dora_tracing::set_up_tracing; -use eyre::{bail, ContextCompat, WrapErr}; +use eyre::{bail, WrapErr}; use std::path::Path; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing("python-operator-dataflow-runner")?; + set_up_tracing("python-dataflow-runner")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - run( - get_python_path().context("Could not get python binary")?, - &["-m", "venv", "../.env"], - None, - ) - .await - .context("failed to create venv")?; - let venv = &root.join("examples").join(".env"); - std::env::set_var( - "VIRTUAL_ENV", - venv.to_str().context("venv path not valid unicode")?, - ); - let orig_path = std::env::var("PATH")?; - // bin folder is named Scripts on windows. - // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 - let venv_bin = if cfg!(windows) { - venv.join("Scripts") - } else { - venv.join("bin") - }; + let uv = get_uv_path().context("Could not get uv binary")?; - if cfg!(windows) { - std::env::set_var( - "PATH", - format!( - "{};{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); - } else { - std::env::set_var( - "PATH", - format!( - "{}:{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); - } + run(&uv, &["venv", "-p", "3.10", "--seed"], None) + .await + .context("failed to create venv")?; - run( - get_python_path().context("Could not get pip binary")?, - &["-m", "pip", "install", "--upgrade", "pip"], - None, - ) - .await - .context("failed to install pip")?; - run( - get_pip_path().context("Could not get pip binary")?, - &["install", "-r", "requirements.txt"], - None, - ) - .await - .context("pip install failed")?; + run(&uv, &["pip", "install", "maturin"], None) + .await + .context("uv pip install maturin failed")?; run( - "maturin", - &["develop"], - Some(&root.join("apis").join("python").join("node")), + &uv, + &[ + "run", + "maturin", + "develop", + "-m", + "../../apis/python/node/Cargo.toml", + "--uv", + ], + None, ) .await .context("maturin develop failed")?; - if std::env::var("CONDA_EXE").is_ok() { - let dataflow = Path::new("dataflow.yml"); - run_dataflow(dataflow).await?; - } else { - let dataflow = Path::new("dataflow_conda.yml"); - run_dataflow(dataflow).await?; - } + let dataflow = Path::new("dataflow.yml"); + run_dataflow(dataflow).await?; Ok(()) } async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); + + // First build the dataflow (install requirements) + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--").arg("build").arg(dataflow).arg("--uv"); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); - cmd.arg("--") - .arg("daemon") - .arg("--run-dataflow") - .arg(dataflow); + cmd.arg("--").arg("run").arg(dataflow).arg("--uv"); if !cmd.status().await?.success() { bail!("failed to run dataflow"); }; diff --git a/examples/python-ros2-dataflow/README.md b/examples/python-ros2-dataflow/README.md index b07f18fc..29894b81 100644 --- a/examples/python-ros2-dataflow/README.md +++ b/examples/python-ros2-dataflow/README.md @@ -7,3 +7,14 @@ source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node & source /opt/ros/humble/setup.bash && ros2 run examples_rclcpp_minimal_service service_main & cargo run --example python-ros2-dataflow --features="ros2-examples" ``` + +- alternatively: + +```bash +source /opt/ros/humble/setup.bash && ros2 run turtlesim turtlesim_node & +source /opt/ros/humble/setup.bash && ros2 run examples_rclcpp_minimal_service service_main & + +# cd examples/python-ros2-dataflow +dora build dataflow.yml --uv +dora run dataflow.yml --uv +``` diff --git a/examples/python-ros2-dataflow/requirements.txt b/examples/python-ros2-dataflow/requirements.txt deleted file mode 100644 index aa0c5e62..00000000 --- a/examples/python-ros2-dataflow/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyarrow -maturin diff --git a/examples/python-ros2-dataflow/run.rs b/examples/python-ros2-dataflow/run.rs index 8eecdf64..d523d5a4 100644 --- a/examples/python-ros2-dataflow/run.rs +++ b/examples/python-ros2-dataflow/run.rs @@ -1,74 +1,37 @@ -use dora_core::{get_pip_path, get_python_path, run}; +use dora_core::{get_uv_path, run}; use dora_tracing::set_up_tracing; -use eyre::{bail, ContextCompat, WrapErr}; +use eyre::{bail, WrapErr}; use std::path::Path; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing("python-ros2-dataflow-runner")?; + set_up_tracing("python-dataflow-runner")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - run( - get_python_path().context("Could not get python binary")?, - &["-m", "venv", "../.env"], - None, - ) - .await - .context("failed to create venv")?; - let venv = &root.join("examples").join(".env"); - std::env::set_var( - "VIRTUAL_ENV", - venv.to_str().context("venv path not valid unicode")?, - ); - let orig_path = std::env::var("PATH")?; - // bin folder is named Scripts on windows. - // 🤦‍♂️ See: https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1 - let venv_bin = if cfg!(windows) { - venv.join("Scripts") - } else { - venv.join("bin") - }; + let uv = get_uv_path().context("Could not get uv binary")?; - if cfg!(windows) { - std::env::set_var( - "PATH", - format!( - "{};{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); - } else { - std::env::set_var( - "PATH", - format!( - "{}:{orig_path}", - venv_bin.to_str().context("venv path not valid unicode")? - ), - ); - } + run(&uv, &["venv", "-p", "3.10", "--seed"], None) + .await + .context("failed to create venv")?; - run( - get_python_path().context("Could not get pip binary")?, - &["-m", "pip", "install", "--upgrade", "pip"], - None, - ) - .await - .context("failed to install pip")?; - run( - get_pip_path().context("Could not get pip binary")?, - &["install", "-r", "requirements.txt"], - None, - ) - .await - .context("pip install failed")?; + run(&uv, &["pip", "install", "maturin"], None) + .await + .context("uv pip install maturin failed")?; run( - "maturin", - &["develop"], - Some(&root.join("apis").join("python").join("node")), + &uv, + &[ + "run", + "maturin", + "develop", + "-m", + "../../apis/python/node/Cargo.toml", + "--uv", + ], + None, ) .await .context("maturin develop failed")?; @@ -81,13 +44,20 @@ async fn main() -> eyre::Result<()> { async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); + + // First build the dataflow (install requirements) + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--").arg("build").arg(dataflow).arg("--uv"); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); - cmd.arg("--") - .arg("daemon") - .arg("--run-dataflow") - .arg(dataflow); + cmd.arg("--").arg("run").arg(dataflow).arg("--uv"); if !cmd.status().await?.success() { bail!("failed to run dataflow"); };