use `download_file` when calling build or start for a url dataflow

Cargo.lock

@@ -2284,6 +2284,7 @@ dependencies = [
  "dora-coordinator",
  "dora-core",
  "dora-daemon",
+ "dora-download",
  "dora-message",
  "dora-node-api-c",
  "dora-operator-api-c",

binaries/cli/Cargo.toml

@@ -24,6 +24,7 @@ dora-core = { workspace = true }
 dora-message = { workspace = true }
 dora-node-api-c = { workspace = true }
 dora-operator-api-c = { workspace = true }
+dora-download = { workspace = true }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_yaml = "0.9.11"
 webbrowser = "0.8.3"

binaries/cli/src/build.rs

@@ -5,8 +5,11 @@ use dora_core::{
 use eyre::{eyre, Context};
 use std::{path::Path, process::Command};
 
-pub fn build(dataflow: &Path) -> eyre::Result<()> {
-    let descriptor = Descriptor::blocking_read(dataflow)?;
+use crate::resolve_dataflow;
+
+pub fn build(dataflow: String) -> eyre::Result<()> {
+    let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
+    let descriptor = Descriptor::blocking_read(&dataflow)?;
     let dataflow_absolute = if dataflow.is_relative() {
         std::env::current_dir().unwrap().join(dataflow)
     } else {

binaries/cli/src/main.rs

@@ -4,13 +4,14 @@ use colored::Colorize;
 use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection};
 use dora_coordinator::Event;
 use dora_core::{
-    descriptor::Descriptor,
+    descriptor::{source_is_url, Descriptor},
     topics::{
         DORA_COORDINATOR_PORT_CONTROL_DEFAULT, DORA_COORDINATOR_PORT_DEFAULT,
         DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT,
     },
 };
 use dora_daemon::Daemon;
+use dora_download::download_file;
 use dora_message::{
     cli_to_coordinator::ControlRequest,
     coordinator_to_cli::{ControlRequestReply, DataflowList, DataflowResult, DataflowStatus},
@@ -21,7 +22,7 @@ use dora_tracing::set_up_tracing_opts;
 use duration_str::parse;
 use eyre::{bail, Context};
 use formatting::FormatDataflowError;
-use std::{io::Write, net::SocketAddr};
+use std::{env::current_dir, io::Write, net::SocketAddr};
 use std::{
     net::{IpAddr, Ipv4Addr},
     path::PathBuf,
@@ -80,8 +81,8 @@ enum Command {
     /// Run build commands provided in the given dataflow.
     Build {
         /// Path to the dataflow descriptor file
-        #[clap(value_name = "PATH", value_hint = clap::ValueHint::FilePath)]
-        dataflow: PathBuf,
+        #[clap(value_name = "PATH")]
+        dataflow: String,
     },
     /// Generate a new project or node. Choose the language between Rust, Python, C or C++.
     New {
@@ -111,8 +112,8 @@ enum Command {
     /// Start the given dataflow path. Attach a name to the running dataflow by using --name.
     Start {
         /// Path to the dataflow descriptor file
-        #[clap(value_name = "PATH", value_hint = clap::ValueHint::FilePath)]
-        dataflow: PathBuf,
+        #[clap(value_name = "PATH")]
+        dataflow: String,
         /// Assign a name to the dataflow
         #[clap(long)]
         name: Option<String>,
@@ -324,7 +325,7 @@ fn run() -> eyre::Result<()> {
             graph::create(dataflow, mermaid, open)?;
         }
         Command::Build { dataflow } => {
-            build::build(&dataflow)?;
+            build::build(dataflow)?;
         }
         Command::New {
             args,
@@ -366,6 +367,7 @@ fn run() -> eyre::Result<()> {
             detach,
             hot_reload,
         } => {
+            let dataflow = resolve_dataflow(dataflow).context("could not resolve dataflow")?;
             let dataflow_descriptor =
                 Descriptor::blocking_read(&dataflow).wrap_err("Failed to read yaml dataflow")?;
             let working_dir = dataflow
@@ -656,3 +658,19 @@ fn connect_to_coordinator(
 ) -> std::io::Result<Box<TcpRequestReplyConnection>> {
     TcpLayer::new().connect(coordinator_addr)
 }
+
+fn resolve_dataflow(dataflow: String) -> eyre::Result<PathBuf> {
+    let dataflow = if source_is_url(&dataflow) {
+        // try to download the shared library
+        let target_path = current_dir().context("Could not access the current dir")?;
+        let rt = Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .context("tokio runtime failed")?;
+        rt.block_on(async { download_file(&dataflow, &target_path).await })
+            .wrap_err("failed to download dataflow yaml file")?
+    } else {
+        PathBuf::from(dataflow)
+    };
+    Ok(dataflow)
+}