Use `run.rs` instead of `run.sh`for better cross-platform usage

examples/python-dataflow/run.rs

@@ -1,46 +1,81 @@
-use eyre::{bail, Context};
-use std::{env, path::Path};
+use eyre::{ContextCompat, WrapErr};
+use std::path::Path;
+use tracing_subscriber::{
+    filter::{FilterExt, LevelFilter},
+    prelude::*,
+    EnvFilter, Registry,
+};
 
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
-    set_up_tracing().wrap_err("failed to set up tracing subscriber")?;
+    set_up_tracing()?;
 
     let root = Path::new(env!("CARGO_MANIFEST_DIR"));
     std::env::set_current_dir(root.join(file!()).parent().unwrap())
         .wrap_err("failed to set working dir")?;
 
-    build_package("dora-daemon").await?;
+    run(&["python3", "-m", "venv", "../.env"], None)
+        .await
+        .context("failed to create venv")?;
+    let venv = &root.join("examples").join(".env");
+    std::env::set_var(
+        "VIRTUAL_ENV",
+        venv.to_str()
+            .context("venv path not valid unicode")?
+            .to_owned(),
+    );
+    let orig_path = std::env::var("PATH")?;
+    let venv_bin = venv.join("bin");
+    std::env::set_var(
+        "PATH",
+        format!(
+            "{}:{orig_path}",
+            venv_bin.to_str().context("venv path not valid unicode")?
+        ),
+    );
 
-    run(root).await?;
+    run(&["pip", "install", "--upgrade", "pip"], None)
+        .await
+        .context("failed to install pip")?;
+    run(&["pip", "install", "-r", "requirements.txt"], None)
+        .await
+        .context("pip install failed")?;
 
-    Ok(())
-}
+    run(
+        &["maturin", "develop"],
+        Some(&root.join("apis").join("python").join("node")),
+    )
+    .await
+    .context("maturin develop failed")?;
+
+    let dataflow = Path::new("dataflow.yml");
+    dora_daemon::Daemon::run_dataflow(dataflow).await?;
 
-async fn build_package(package: &str) -> eyre::Result<()> {
-    let cargo = std::env::var("CARGO").unwrap();
-    let mut cmd = tokio::process::Command::new(&cargo);
-    cmd.arg("build");
-    cmd.arg("--package").arg(package);
-    if !cmd.status().await?.success() {
-        bail!("failed to build {package}");
-    };
     Ok(())
 }
 
-async fn run(_root: &Path) -> eyre::Result<()> {
-    let mut run = tokio::process::Command::new("sh");
-    run.arg("./run.sh");
+async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> {
+    let mut run = tokio::process::Command::new(cmd[0]);
+    run.args(&cmd[1..]);
+
+    if let Some(pwd) = pwd {
+        run.current_dir(pwd);
+    }
     if !run.status().await?.success() {
-        bail!("failed to run python example.");
+        eyre::bail!("failed to run {cmd:?}");
     };
     Ok(())
 }
 
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
+pub fn set_up_tracing() -> eyre::Result<()> {
+    // Filter log using `RUST_LOG`. More useful for CLI.
+    let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG);
+    let stdout_log = tracing_subscriber::fmt::layer()
+        .pretty()
+        .with_filter(filter);
+
+    let registry = Registry::default().with(stdout_log);
 
-    let stdout_log = tracing_subscriber::fmt::layer().pretty();
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
+    tracing::subscriber::set_global_default(registry)
         .context("failed to set tracing global subscriber")
 }

examples/python-dataflow/run.sh

@@ -1,11 +0,0 @@
-set -e
-
-python3 -m venv ../.env
-. $(pwd)/../.env/bin/activate
-
-# Dependencies
-pip install --upgrade pip
-pip install -r requirements.txt
-maturin develop -m ../../apis/python/node/Cargo.toml
-
-cargo run -p dora-daemon -- --run-dataflow dataflow.yml

examples/python-operator-dataflow/run.rs

@@ -1,46 +1,81 @@
-use eyre::{bail, Context};
-use std::{env, path::Path};
+use eyre::{ContextCompat, WrapErr};
+use std::path::Path;
+use tracing_subscriber::{
+    filter::{FilterExt, LevelFilter},
+    prelude::*,
+    EnvFilter, Registry,
+};
 
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
-    set_up_tracing().wrap_err("failed to set up tracing subscriber")?;
+    set_up_tracing()?;
 
     let root = Path::new(env!("CARGO_MANIFEST_DIR"));
     std::env::set_current_dir(root.join(file!()).parent().unwrap())
         .wrap_err("failed to set working dir")?;
 
-    build_package("dora-daemon").await?;
+    run(&["python3", "-m", "venv", "../.env"], None)
+        .await
+        .context("failed to create venv")?;
+    let venv = &root.join("examples").join(".env");
+    std::env::set_var(
+        "VIRTUAL_ENV",
+        venv.to_str()
+            .context("venv path not valid unicode")?
+            .to_owned(),
+    );
+    let orig_path = std::env::var("PATH")?;
+    let venv_bin = venv.join("bin");
+    std::env::set_var(
+        "PATH",
+        format!(
+            "{}:{orig_path}",
+            venv_bin.to_str().context("venv path not valid unicode")?
+        ),
+    );
 
-    run(root).await?;
+    run(&["pip", "install", "--upgrade", "pip"], None)
+        .await
+        .context("failed to install pip")?;
+    run(&["pip", "install", "-r", "requirements.txt"], None)
+        .await
+        .context("pip install failed")?;
 
-    Ok(())
-}
+    run(
+        &["maturin", "develop"],
+        Some(&root.join("apis").join("python").join("node")),
+    )
+    .await
+    .context("maturin develop failed")?;
+
+    let dataflow = Path::new("dataflow.yml");
+    dora_daemon::Daemon::run_dataflow(dataflow).await?;
 
-async fn build_package(package: &str) -> eyre::Result<()> {
-    let cargo = std::env::var("CARGO").unwrap();
-    let mut cmd = tokio::process::Command::new(&cargo);
-    cmd.arg("build");
-    cmd.arg("--package").arg(package);
-    if !cmd.status().await?.success() {
-        bail!("failed to build {package}");
-    };
     Ok(())
 }
 
-async fn run(_root: &Path) -> eyre::Result<()> {
-    let mut run = tokio::process::Command::new("sh");
-    run.arg("./run.sh");
+async fn run(cmd: &[&str], pwd: Option<&Path>) -> eyre::Result<()> {
+    let mut run = tokio::process::Command::new(cmd[0]);
+    run.args(&cmd[1..]);
+
+    if let Some(pwd) = pwd {
+        run.current_dir(pwd);
+    }
     if !run.status().await?.success() {
-        bail!("failed to run python example.");
+        eyre::bail!("failed to run {cmd:?}");
     };
     Ok(())
 }
 
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
+pub fn set_up_tracing() -> eyre::Result<()> {
+    // Filter log using `RUST_LOG`. More useful for CLI.
+    let filter = EnvFilter::from_default_env().or(LevelFilter::DEBUG);
+    let stdout_log = tracing_subscriber::fmt::layer()
+        .pretty()
+        .with_filter(filter);
+
+    let registry = Registry::default().with(stdout_log);
 
-    let stdout_log = tracing_subscriber::fmt::layer().pretty();
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
+    tracing::subscriber::set_global_default(registry)
         .context("failed to set tracing global subscriber")
 }

examples/python-operator-dataflow/run.sh

@@ -1,11 +0,0 @@
-set -e
-
-python3 -m venv .env
-. $(pwd)/.env/bin/activate
-
-# Dependencies
-pip install --upgrade pip
-pip install -r requirements.txt
-maturin develop -m ../../apis/python/node/Cargo.toml
-
-cargo run -p dora-daemon -- --run-dataflow dataflow.yml