From 942d0a5df2d84ff19417ced16c8dd8568eab4d63 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 6 Oct 2023 15:34:02 +0200 Subject: [PATCH] Use `run.rs` instead of `run.sh`for better cross-platform usage --- examples/python-dataflow/run.rs | 83 +++++++++++++++++------- examples/python-dataflow/run.sh | 11 ---- examples/python-operator-dataflow/run.rs | 83 +++++++++++++++++------- examples/python-operator-dataflow/run.sh | 11 ---- 4 files changed, 118 insertions(+), 70 deletions(-) delete mode 100644 examples/python-dataflow/run.sh delete mode 100644 examples/python-operator-dataflow/run.sh diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index ac32ff00..e531b8b2 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,46 +1,81 @@ -use eyre::{bail, Context}; -use std::{env, path::Path}; +use eyre::{ContextCompat, WrapErr}; +use std::path::Path; +use tracing_subscriber::{ + filter::{FilterExt, LevelFilter}, + prelude::*, + EnvFilter, Registry, +}; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing()?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - build_package("dora-daemon").await?; + run(&["python3", "-m", "venv", "../.env"], None) + .await + .context("failed to create venv")?; + let venv = &root.join("examples").join(".env"); + std::env::set_var( + "VIRTUAL_ENV", + venv.to_str() + .context("venv path not valid unicode")? + .to_owned(), + ); + let orig_path = std::env::var("PATH")?; + let venv_bin = venv.join("bin"); + std::env::set_var( + "PATH", + format!( + "{}:{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); - run(root).await?; + run(&["pip", "install", "--upgrade", "pip"], None) + .await + .context("failed to install pip")?; + run(&["pip", "install", "-r", "requirements.txt"], None) + .await + .context("pip install failed")?; - Ok(()) -} + run( + &["maturin", "develop"], + Some(&root.join("apis").join("python").join("node")), + ) + .await + .context("maturin develop failed")?; + + let dataflow = Path::new("dataflow.yml"); + dora_daemon::Daemon::run_dataflow(dataflow).await?; -async fn build_package(package: &str) -> eyre::Result<()> { - let cargo = std::env::var("CARGO").unwrap(); - let mut cmd = tokio::process::Command::new(&cargo); - cmd.arg("build"); - cmd.arg("--package").arg(package); - if !cmd.status().await?.success() { - bail!("failed to build {package}"); - }; Ok(()) } -async fn run(_root: &Path) -> eyre::Result<()> { - let mut run = tokio::process::Command::new("sh"); - run.arg("./run.sh"); +async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> { + let mut run = tokio::process::Command::new(cmd[0]); + run.args(&cmd[1..]); + + if let Some(pwd) = pwd { + run.current_dir(pwd); + } if !run.status().await?.success() { - bail!("failed to run python example."); + eyre::bail!("failed to run {cmd:?}"); }; Ok(()) } -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; +pub fn set_up_tracing() -> eyre::Result<()> { + // Filter log using `RUST_LOG`. More useful for CLI. + let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG); + let stdout_log = tracing_subscriber::fmt::layer() + .pretty() + .with_filter(filter); + + let registry = Registry::default().with(stdout_log); - let stdout_log = tracing_subscriber::fmt::layer().pretty(); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) + tracing::subscriber::set_global_default(registry) .context("failed to set tracing global subscriber") } diff --git a/examples/python-dataflow/run.sh b/examples/python-dataflow/run.sh deleted file mode 100644 index a967d253..00000000 --- a/examples/python-dataflow/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -set -e - -python3 -m venv ../.env -. $(pwd)/../.env/bin/activate - -# Dependencies -pip install --upgrade pip -pip install -r requirements.txt -maturin develop -m ../../apis/python/node/Cargo.toml - -cargo run -p dora-daemon -- --run-dataflow dataflow.yml diff --git a/examples/python-operator-dataflow/run.rs b/examples/python-operator-dataflow/run.rs index ac32ff00..e531b8b2 100644 --- a/examples/python-operator-dataflow/run.rs +++ b/examples/python-operator-dataflow/run.rs @@ -1,46 +1,81 @@ -use eyre::{bail, Context}; -use std::{env, path::Path}; +use eyre::{ContextCompat, WrapErr}; +use std::path::Path; +use tracing_subscriber::{ + filter::{FilterExt, LevelFilter}, + prelude::*, + EnvFilter, Registry, +}; #[tokio::main] async fn main() -> eyre::Result<()> { - set_up_tracing().wrap_err("failed to set up tracing subscriber")?; + set_up_tracing()?; let root = Path::new(env!("CARGO_MANIFEST_DIR")); std::env::set_current_dir(root.join(file!()).parent().unwrap()) .wrap_err("failed to set working dir")?; - build_package("dora-daemon").await?; + run(&["python3", "-m", "venv", "../.env"], None) + .await + .context("failed to create venv")?; + let venv = &root.join("examples").join(".env"); + std::env::set_var( + "VIRTUAL_ENV", + venv.to_str() + .context("venv path not valid unicode")? + .to_owned(), + ); + let orig_path = std::env::var("PATH")?; + let venv_bin = venv.join("bin"); + std::env::set_var( + "PATH", + format!( + "{}:{orig_path}", + venv_bin.to_str().context("venv path not valid unicode")? + ), + ); - run(root).await?; + run(&["pip", "install", "--upgrade", "pip"], None) + .await + .context("failed to install pip")?; + run(&["pip", "install", "-r", "requirements.txt"], None) + .await + .context("pip install failed")?; - Ok(()) -} + run( + &["maturin", "develop"], + Some(&root.join("apis").join("python").join("node")), + ) + .await + .context("maturin develop failed")?; + + let dataflow = Path::new("dataflow.yml"); + dora_daemon::Daemon::run_dataflow(dataflow).await?; -async fn build_package(package: &str) -> eyre::Result<()> { - let cargo = std::env::var("CARGO").unwrap(); - let mut cmd = tokio::process::Command::new(&cargo); - cmd.arg("build"); - cmd.arg("--package").arg(package); - if !cmd.status().await?.success() { - bail!("failed to build {package}"); - }; Ok(()) } -async fn run(_root: &Path) -> eyre::Result<()> { - let mut run = tokio::process::Command::new("sh"); - run.arg("./run.sh"); +async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> { + let mut run = tokio::process::Command::new(cmd[0]); + run.args(&cmd[1..]); + + if let Some(pwd) = pwd { + run.current_dir(pwd); + } if !run.status().await?.success() { - bail!("failed to run python example."); + eyre::bail!("failed to run {cmd:?}"); }; Ok(()) } -fn set_up_tracing() -> eyre::Result<()> { - use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; +pub fn set_up_tracing() -> eyre::Result<()> { + // Filter log using `RUST_LOG`. More useful for CLI. + let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG); + let stdout_log = tracing_subscriber::fmt::layer() + .pretty() + .with_filter(filter); + + let registry = Registry::default().with(stdout_log); - let stdout_log = tracing_subscriber::fmt::layer().pretty(); - let subscriber = tracing_subscriber::Registry::default().with(stdout_log); - tracing::subscriber::set_global_default(subscriber) + tracing::subscriber::set_global_default(registry) .context("failed to set tracing global subscriber") } diff --git a/examples/python-operator-dataflow/run.sh b/examples/python-operator-dataflow/run.sh deleted file mode 100644 index f32ff366..00000000 --- a/examples/python-operator-dataflow/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -set -e - -python3 -m venv .env -. $(pwd)/.env/bin/activate - -# Dependencies -pip install --upgrade pip -pip install -r requirements.txt -maturin develop -m ../../apis/python/node/Cargo.toml - -cargo run -p dora-daemon -- --run-dataflow dataflow.yml