From 64a2dc9c19ff6a4ac355c16ca175722785e2d260 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 6 Jun 2025 18:00:05 +0200 Subject: [PATCH] Wip --- Cargo.lock | 225 +++++++++++++----------- Cargo.toml | 1 + binaries/cli/Cargo.toml | 4 +- binaries/cli/src/command/build/local.rs | 25 +-- binaries/daemon/Cargo.toml | 2 +- binaries/daemon/src/lib.rs | 25 +-- examples/rust-dataflow-git/.gitignore | 1 + examples/rust-dataflow-git/dataflow.yml | 10 +- libraries/core/Cargo.toml | 2 +- libraries/core/src/build/git.rs | 81 ++++++--- 10 files changed, 201 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00c290c8..9067383e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -851,7 +851,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror 1.0.69", - "time 0.3.41", + "time", ] [[package]] @@ -1415,7 +1415,7 @@ dependencies = [ "path_abs", "plist", "regex", - "semver 1.0.26", + "semver", "serde", "serde_yaml 0.9.34+deprecated", "shell-words", @@ -1823,7 +1823,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.26", + "semver", "serde", "serde_json", ] @@ -1836,7 +1836,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" dependencies = [ "camino", "cargo-platform", - "semver 1.0.26", + "semver", "serde", "serde_json", "thiserror 1.0.69", @@ -2521,6 +2521,33 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "cxx" version = "1.0.149" @@ -3115,7 +3142,7 @@ dependencies = [ "log", "once_cell", "schemars", - "semver 1.0.26", + "semver", "serde", "serde-with-expand-env", "serde_yaml 0.9.34+deprecated", @@ -3462,7 +3489,7 @@ dependencies = [ "rust_decimal", "serde", "thiserror 1.0.69", - "time 0.3.41", + "time", ] [[package]] @@ -3499,6 +3526,31 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18aade80d5e09429040243ce1143ddc08a92d7a22820ac512610410a4dd5214f" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "signature 2.2.0", + "subtle", + "zeroize", +] + [[package]] name = "eframe" version = "0.31.1" @@ -4106,6 +4158,12 @@ dependencies = [ "anyhow", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.25" @@ -5080,7 +5138,7 @@ dependencies = [ "dirs 5.0.1", "futures", "http 1.3.1", - "indicatif 0.17.11", + "indicatif", "libc", "log", "num_cpus", @@ -5211,7 +5269,7 @@ dependencies = [ "http 0.2.12", "http-serde", "serde", - "time 0.3.41", + "time", ] [[package]] @@ -5609,18 +5667,6 @@ dependencies = [ "serde", ] -[[package]] -name = "indicatif" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" -dependencies = [ - "console", - "lazy_static", - "number_prefix 0.3.0", - "regex", -] - [[package]] name = "indicatif" version = "0.17.11" @@ -5628,7 +5674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", - "number_prefix 0.4.0", + "number_prefix", "portable-atomic", "rayon", "unicode-width 0.2.0", @@ -6737,7 +6783,7 @@ dependencies = [ "hf-hub", "image", "indexmap 2.8.0", - "indicatif 0.17.11", + "indicatif", "interprocess", "itertools 0.13.0", "llguidance", @@ -7374,12 +7420,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" - [[package]] name = "number_prefix" version = "0.4.0" @@ -8389,7 +8429,7 @@ dependencies = [ "indexmap 2.8.0", "quick-xml 0.32.0", "serde", - "time 0.3.41", + "time", ] [[package]] @@ -8887,15 +8927,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" -[[package]] -name = "quick-xml" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.30.0" @@ -9245,7 +9276,7 @@ dependencies = [ "serde_json", "sha2", "thiserror 1.0.69", - "time 0.3.41", + "time", "url", "uuid 1.16.0", "web-sys", @@ -9332,7 +9363,7 @@ dependencies = [ "cargo_metadata 0.18.1", "glob", "sha2", - "time 0.3.41", + "time", "unindent", "walkdir", ] @@ -9767,7 +9798,7 @@ dependencies = [ "serde_bytes", "static_assertions", "thiserror 1.0.69", - "time 0.3.41", + "time", "typenum", "uuid 1.16.0", "web-time", @@ -10248,7 +10279,7 @@ dependencies = [ "strum 0.26.3", "strum_macros 0.26.4", "sublime_fuzzy", - "time 0.3.41", + "time", "url", ] @@ -11334,7 +11365,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.26", + "semver", ] [[package]] @@ -11750,34 +11781,39 @@ dependencies = [ "libc", ] +[[package]] +name = "self-replace" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ec815b5eab420ab893f63393878d89c90fdd94c0bcc44c07abb8ad95552fb7" +dependencies = [ + "fastrand 2.3.0", + "tempfile", + "windows-sys 0.52.0", +] + [[package]] name = "self_update" -version = "0.27.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb85f1802f7b987237b8525c0fde86ea86f31c957c1875467c727d5b921179c" +checksum = "d832c086ece0dacc29fb2947bb4219b8f6e12fe9e40b7108f9e57c4224e47b5c" dependencies = [ "either", "flate2", - "hyper 0.14.32", - "indicatif 0.15.0", + "hyper 1.6.0", + "indicatif", "log", - "quick-xml 0.20.0", + "quick-xml 0.37.2", "regex", - "reqwest 0.11.27", - "semver 0.11.0", + "reqwest 0.12.15", + "self-replace", + "semver", "serde_json", "tar", "tempfile", - "zip 0.5.13", -] - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser", + "urlencoding", + "zip 2.4.2", + "zipsign-api", ] [[package]] @@ -11789,15 +11825,6 @@ dependencies = [ "serde", ] -[[package]] -name = "semver-parser" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9900206b54a3527fdc7b8a938bffd94a568bac4f4aa8113b209df75a09c0dec2" -dependencies = [ - "pest", -] - [[package]] name = "seq-macro" version = "0.3.6" @@ -11964,7 +11991,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_with_macros", - "time 0.3.41", + "time", ] [[package]] @@ -13022,17 +13049,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", -] - [[package]] name = "time" version = "0.3.41" @@ -13195,7 +13211,7 @@ dependencies = [ "derive_builder", "esaxx-rs", "getrandom 0.2.15", - "indicatif 0.17.11", + "indicatif", "itertools 0.13.0", "lazy_static", "log", @@ -14081,12 +14097,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -15374,7 +15384,7 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror 1.0.69", - "time 0.3.41", + "time", ] [[package]] @@ -15986,7 +15996,7 @@ dependencies = [ "rustls 0.23.25", "rustls-webpki 0.102.8", "serde", - "time 0.3.41", + "time", "tokio", "tokio-util", "tracing", @@ -16037,7 +16047,7 @@ dependencies = [ "rustls-pki-types", "rustls-webpki 0.102.8", "secrecy", - "time 0.3.41", + "time", "tokio", "tokio-util", "tracing", @@ -16122,7 +16132,7 @@ dependencies = [ "rustls-webpki 0.102.8", "secrecy", "socket2 0.5.8", - "time 0.3.41", + "time", "tls-listener", "tokio", "tokio-rustls 0.26.2", @@ -16608,29 +16618,44 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.13" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" dependencies = [ - "byteorder", + "arbitrary", "crc32fast", + "crossbeam-utils", + "displaydoc", + "indexmap 2.8.0", + "num_enum", "thiserror 1.0.69", - "time 0.1.45", ] [[package]] name = "zip" -version = "1.1.4" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" dependencies = [ "arbitrary", "crc32fast", "crossbeam-utils", "displaydoc", "indexmap 2.8.0", - "num_enum", - "thiserror 1.0.69", + "memchr", + "thiserror 2.0.12", + "time", +] + +[[package]] +name = "zipsign-api" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba6063ff82cdbd9a765add16d369abe81e520f836054e997c2db217ceca40c0" +dependencies = [ + "base64 0.22.1", + "ed25519-dalek", + "thiserror 2.0.12", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8ae578fb..d2489a52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ pyo3 = { version = "0.23", features = [ "multiple-pymethods", ] } pythonize = "0.23" +git2 = { version = "0.18.0", features = ["vendored-openssl"] } [package] name = "dora-examples" diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index 11896d96..96b67a76 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -50,7 +50,7 @@ tabwriter = "1.4.0" log = { version = "0.4.21", features = ["serde"] } colored = "2.1.0" env_logger = "0.11.3" -self_update = { version = "0.27.0", features = [ +self_update = { version = "0.42.0", features = [ "rustls", "archive-zip", "archive-tar", @@ -61,7 +61,7 @@ pyo3 = { workspace = true, features = [ "abi3", ], optional = true } dunce = "1.0.5" -git2 = { version = "0.18.0", features = ["vendored-openssl"] } +git2 = { workspace = true } [lib] name = "dora_cli" diff --git a/binaries/cli/src/command/build/local.rs b/binaries/cli/src/command/build/local.rs index 05695fd7..ac28eeca 100644 --- a/binaries/cli/src/command/build/local.rs +++ b/binaries/cli/src/command/build/local.rs @@ -71,27 +71,14 @@ async fn build_dataflow( let mut info = BuildInfo { node_working_dirs: Default::default(), }; - let mut errors = Vec::new(); for (node_id, task) in tasks { - match task.await { - Ok(node) => { - info.node_working_dirs - .insert(node_id, node.node_working_dir); - } - Err(err) => { - errors.push((node_id, err)); - } - } - } - if errors.is_empty() { - Ok(info) - } else { - let mut message = "failed to build dataflow:\n".to_owned(); - for (node_id, err) in errors { - message.push_str(&format!("- {node_id}: {err:?}\n-------------------\n\n")); - } - Err(eyre::eyre!(message)) + let node = task + .await + .with_context(|| format!("failed to build node `{node_id}`"))?; + info.node_working_dirs + .insert(node_id, node.node_working_dir); } + Ok(info) } struct LocalBuildLogger { diff --git a/binaries/daemon/Cargo.toml b/binaries/daemon/Cargo.toml index 6dd2a2c3..fdfd3596 100644 --- a/binaries/daemon/Cargo.toml +++ b/binaries/daemon/Cargo.toml @@ -45,6 +45,6 @@ crossbeam = "0.8.4" crossbeam-skiplist = "0.1.3" zenoh = "1.1.1" url = "2.5.4" -git2 = { version = "0.18.0", features = ["vendored-openssl"] } +git2 = { workspace = true } dunce = "1.0.5" itertools = "0.14" diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index ba724680..f7a16c9a 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -946,32 +946,19 @@ impl Daemon { let mut info = BuildInfo { node_working_dirs: Default::default(), }; - let mut errors = Vec::new(); for task in tasks { let NodeBuildTask { node_id, dynamic_node, task, } = task; - match task.await { - Ok(node) => { - info.node_working_dirs - .insert(node_id, node.node_working_dir); - } - Err(err) => { - errors.push((node_id, err)); - } - } - } - if errors.is_empty() { - Ok(info) - } else { - let mut message = "failed to build dataflow:\n".to_owned(); - for (node_id, err) in errors { - message.push_str(&format!("- {node_id}: {err:?}\n-------------------\n\n")); - } - Err(eyre::eyre!(message)) + let node = task + .await + .with_context(|| format!("failed to build node `{node_id}`"))?; + info.node_working_dirs + .insert(node_id, node.node_working_dir); } + Ok(info) }; Ok(task) diff --git a/examples/rust-dataflow-git/.gitignore b/examples/rust-dataflow-git/.gitignore index 796b96d1..dfdc87e3 100644 --- a/examples/rust-dataflow-git/.gitignore +++ b/examples/rust-dataflow-git/.gitignore @@ -1 +1,2 @@ /build +/git diff --git a/examples/rust-dataflow-git/dataflow.yml b/examples/rust-dataflow-git/dataflow.yml index f1fa7512..eccacdeb 100644 --- a/examples/rust-dataflow-git/dataflow.yml +++ b/examples/rust-dataflow-git/dataflow.yml @@ -1,9 +1,9 @@ nodes: - id: rust-node git: https://github.com/dora-rs/dora.git - rev: e31b2a34 # pinned commit, update this when changing the message crate - build: cargo build --manifest-path build/github.com/dora-rs/dora.git/e31b2a34/Cargo.toml -p rust-dataflow-example-node - path: build/github.com/dora-rs/dora.git/e31b2a34/target/debug/rust-dataflow-example-node + rev: a8c55b5a # pinned commit, update this when changing the message crate + build: cargo build -p rust-dataflow-example-node + path: target/debug/rust-dataflow-example-node inputs: tick: dora/timer/millis/10 outputs: @@ -11,7 +11,7 @@ nodes: - id: rust-status-node git: https://github.com/dora-rs/dora.git - rev: e31b2a34 # pinned commit, update this when changing the message crate + rev: a8c55b5a # pinned commit, update this when changing the message crate build: cargo build -p rust-dataflow-example-status-node path: target/debug/rust-dataflow-example-status-node inputs: @@ -22,7 +22,7 @@ nodes: - id: rust-sink git: https://github.com/dora-rs/dora.git - rev: e31b2a34 # pinned commit, update this when changing the message crate + rev: a8c55b5a # pinned commit, update this when changing the message crate build: cargo build -p rust-dataflow-example-sink path: target/debug/rust-dataflow-example-sink inputs: diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml index bb6cb6ec..af467ffe 100644 --- a/libraries/core/Cargo.toml +++ b/libraries/core/Cargo.toml @@ -25,5 +25,5 @@ serde_json = "1.0.117" log = { version = "0.4.21", features = ["serde"] } dunce = "1.0.5" url = "2.5.4" -git2 = { version = "0.18.0", features = ["vendored-openssl"] } +git2 = { workspace = true } itertools = "0.14" diff --git a/libraries/core/src/build/git.rs b/libraries/core/src/build/git.rs index d62dfd94..f53a5c9e 100644 --- a/libraries/core/src/build/git.rs +++ b/libraries/core/src/build/git.rs @@ -1,6 +1,6 @@ use crate::build::BuildLogger; use dora_message::{common::LogLevel, descriptor::GitRepoRev, DataflowId, SessionId}; -use eyre::{ContextCompat, WrapErr}; +use eyre::{bail, ContextCompat, WrapErr}; use git2::FetchOptions; use itertools::Itertools; use std::{ @@ -145,15 +145,53 @@ impl GitFolder { pub async fn prepare(self, logger: &mut impl BuildLogger) -> eyre::Result { let GitFolder { reuse } = self; + eprintln!("reuse: {reuse:?}"); let clone_dir = match reuse { ReuseOptions::NewClone { target_dir, repo_url, commit_hash, } => { - let repository = clone_into(repo_url, &target_dir, logger).await?; - checkout_tree(&repository, &commit_hash)?; - target_dir + logger + .log_message( + LogLevel::Info, + format!( + "cloning {repo_url}#{commit_hash} into {}", + target_dir.display() + ), + ) + .await; + let clone_target = target_dir.clone(); + let checkout_result = tokio::task::spawn_blocking(move || { + let repository = clone_into(repo_url.clone(), &clone_target) + .with_context(|| format!("failed to clone git repo from `{repo_url}`"))?; + checkout_tree(&repository, &commit_hash) + .with_context(|| format!("failed to checkout commit `{commit_hash}`")) + }) + .await + .unwrap(); + + match checkout_result { + Ok(()) => target_dir, + Err(err) => { + logger + .log_message(LogLevel::Error, format!("{err:?}")) + .await; + // remove erroneous clone again + if let Err(err) = std::fs::remove_dir_all(target_dir) { + logger + .log_message( + LogLevel::Error, + format!( + "failed to remove clone dir after clone/checkout error: {}", + err.kind() + ), + ) + .await; + } + bail!(err) + } + } } ReuseOptions::CopyAndFetch { from, @@ -209,6 +247,7 @@ impl GitFolder { } } +#[derive(Debug)] enum ReuseOptions { /// Create a new clone of the repository. NewClone { @@ -241,35 +280,21 @@ fn rev_str(rev: &Option) -> String { } } -async fn clone_into( - repo_addr: Url, - clone_dir: &Path, - logger: &mut impl BuildLogger, -) -> eyre::Result { +fn clone_into(repo_addr: Url, clone_dir: &Path) -> eyre::Result { if let Some(parent) = clone_dir.parent() { - tokio::fs::create_dir_all(parent) - .await + std::fs::create_dir_all(parent) .context("failed to create parent directory for git clone")?; } - logger - .log_message( - LogLevel::Info, - format!("cloning {repo_addr} into {}", clone_dir.display()), - ) - .await; let clone_dir = clone_dir.to_owned(); - let task = tokio::task::spawn_blocking(move || { - let mut builder = git2::build::RepoBuilder::new(); - let mut fetch_options = git2::FetchOptions::new(); - fetch_options.download_tags(git2::AutotagOption::All); - builder.fetch_options(fetch_options); - builder - .clone(repo_addr.as_str(), &clone_dir) - .context("failed to clone repo") - }); - let repo = task.await??; - Ok(repo) + + let mut builder = git2::build::RepoBuilder::new(); + let mut fetch_options = git2::FetchOptions::new(); + fetch_options.download_tags(git2::AutotagOption::All); + builder.fetch_options(fetch_options); + builder + .clone(repo_addr.as_str(), &clone_dir) + .context("failed to clone repo") } async fn fetch_changes(