dora-rs
/
dora
Not watched
1
0
Fork 0
Code
Releases 124 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Wip

wip
Philipp Oppermann 1 year ago
parent
a8c55b5a23
commit
64a2dc9c19
Failed to extract signature
10 changed files with 201 additions and 175 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +125
    -100
      Cargo.lock
  2. +1
    -0
      Cargo.toml
  3. +2
    -2
      binaries/cli/Cargo.toml
  4. +6
    -19
      binaries/cli/src/command/build/local.rs
  5. +1
    -1
      binaries/daemon/Cargo.toml
  6. +6
    -19
      binaries/daemon/src/lib.rs
  7. +1
    -0
      examples/rust-dataflow-git/.gitignore
  8. +5
    -5
      examples/rust-dataflow-git/dataflow.yml
  9. +1
    -1
      libraries/core/Cargo.toml
  10. +53
    -28
      libraries/core/src/build/git.rs

+ 125
- 100
Cargo.lock View File

@@ -851,7 +851,7 @@ dependencies = [
"num-traits",
"rusticata-macros",
"thiserror 1.0.69",
"time 0.3.41",
"time",
]

[[package]]
@@ -1415,7 +1415,7 @@ dependencies = [
"path_abs",
"plist",
"regex",
"semver 1.0.26",
"semver",
"serde",
"serde_yaml 0.9.34+deprecated",
"shell-words",
@@ -1823,7 +1823,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
dependencies = [
"camino",
"cargo-platform",
"semver 1.0.26",
"semver",
"serde",
"serde_json",
]
@@ -1836,7 +1836,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
dependencies = [
"camino",
"cargo-platform",
"semver 1.0.26",
"semver",
"serde",
"serde_json",
"thiserror 1.0.69",
@@ -2521,6 +2521,33 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991"

[[package]]
name = "curve25519-dalek"
version = "4.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
dependencies = [
"cfg-if 1.0.0",
"cpufeatures",
"curve25519-dalek-derive",
"digest",
"fiat-crypto",
"rustc_version",
"subtle",
"zeroize",
]

[[package]]
name = "curve25519-dalek-derive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]

[[package]]
name = "cxx"
version = "1.0.149"
@@ -3115,7 +3142,7 @@ dependencies = [
"log",
"once_cell",
"schemars",
"semver 1.0.26",
"semver",
"serde",
"serde-with-expand-env",
"serde_yaml 0.9.34+deprecated",
@@ -3462,7 +3489,7 @@ dependencies = [
"rust_decimal",
"serde",
"thiserror 1.0.69",
"time 0.3.41",
"time",
]

[[package]]
@@ -3499,6 +3526,31 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18aade80d5e09429040243ce1143ddc08a92d7a22820ac512610410a4dd5214f"

[[package]]
name = "ed25519"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
dependencies = [
"pkcs8 0.10.2",
"signature 2.2.0",
]

[[package]]
name = "ed25519-dalek"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
dependencies = [
"curve25519-dalek",
"ed25519",
"serde",
"sha2",
"signature 2.2.0",
"subtle",
"zeroize",
]

[[package]]
name = "eframe"
version = "0.31.1"
@@ -4106,6 +4158,12 @@ dependencies = [
"anyhow",
]

[[package]]
name = "fiat-crypto"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"

[[package]]
name = "filetime"
version = "0.2.25"
@@ -5080,7 +5138,7 @@ dependencies = [
"dirs 5.0.1",
"futures",
"http 1.3.1",
"indicatif 0.17.11",
"indicatif",
"libc",
"log",
"num_cpus",
@@ -5211,7 +5269,7 @@ dependencies = [
"http 0.2.12",
"http-serde",
"serde",
"time 0.3.41",
"time",
]

[[package]]
@@ -5609,18 +5667,6 @@ dependencies = [
"serde",
]

[[package]]
name = "indicatif"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
dependencies = [
"console",
"lazy_static",
"number_prefix 0.3.0",
"regex",
]

[[package]]
name = "indicatif"
version = "0.17.11"
@@ -5628,7 +5674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
dependencies = [
"console",
"number_prefix 0.4.0",
"number_prefix",
"portable-atomic",
"rayon",
"unicode-width 0.2.0",
@@ -6737,7 +6783,7 @@ dependencies = [
"hf-hub",
"image",
"indexmap 2.8.0",
"indicatif 0.17.11",
"indicatif",
"interprocess",
"itertools 0.13.0",
"llguidance",
@@ -7374,12 +7420,6 @@ dependencies = [
"libc",
]

[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"

[[package]]
name = "number_prefix"
version = "0.4.0"
@@ -8389,7 +8429,7 @@ dependencies = [
"indexmap 2.8.0",
"quick-xml 0.32.0",
"serde",
"time 0.3.41",
"time",
]

[[package]]
@@ -8887,15 +8927,6 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"

[[package]]
name = "quick-xml"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd"
dependencies = [
"memchr",
]

[[package]]
name = "quick-xml"
version = "0.30.0"
@@ -9245,7 +9276,7 @@ dependencies = [
"serde_json",
"sha2",
"thiserror 1.0.69",
"time 0.3.41",
"time",
"url",
"uuid 1.16.0",
"web-sys",
@@ -9332,7 +9363,7 @@ dependencies = [
"cargo_metadata 0.18.1",
"glob",
"sha2",
"time 0.3.41",
"time",
"unindent",
"walkdir",
]
@@ -9767,7 +9798,7 @@ dependencies = [
"serde_bytes",
"static_assertions",
"thiserror 1.0.69",
"time 0.3.41",
"time",
"typenum",
"uuid 1.16.0",
"web-time",
@@ -10248,7 +10279,7 @@ dependencies = [
"strum 0.26.3",
"strum_macros 0.26.4",
"sublime_fuzzy",
"time 0.3.41",
"time",
"url",
]

@@ -11334,7 +11365,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver 1.0.26",
"semver",
]

[[package]]
@@ -11750,34 +11781,39 @@ dependencies = [
"libc",
]

[[package]]
name = "self-replace"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03ec815b5eab420ab893f63393878d89c90fdd94c0bcc44c07abb8ad95552fb7"
dependencies = [
"fastrand 2.3.0",
"tempfile",
"windows-sys 0.52.0",
]

[[package]]
name = "self_update"
version = "0.27.0"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb85f1802f7b987237b8525c0fde86ea86f31c957c1875467c727d5b921179c"
checksum = "d832c086ece0dacc29fb2947bb4219b8f6e12fe9e40b7108f9e57c4224e47b5c"
dependencies = [
"either",
"flate2",
"hyper 0.14.32",
"indicatif 0.15.0",
"hyper 1.6.0",
"indicatif",
"log",
"quick-xml 0.20.0",
"quick-xml 0.37.2",
"regex",
"reqwest 0.11.27",
"semver 0.11.0",
"reqwest 0.12.15",
"self-replace",
"semver",
"serde_json",
"tar",
"tempfile",
"zip 0.5.13",
]

[[package]]
name = "semver"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
dependencies = [
"semver-parser",
"urlencoding",
"zip 2.4.2",
"zipsign-api",
]

[[package]]
@@ -11789,15 +11825,6 @@ dependencies = [
"serde",
]

[[package]]
name = "semver-parser"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9900206b54a3527fdc7b8a938bffd94a568bac4f4aa8113b209df75a09c0dec2"
dependencies = [
"pest",
]

[[package]]
name = "seq-macro"
version = "0.3.6"
@@ -11964,7 +11991,7 @@ dependencies = [
"serde_derive",
"serde_json",
"serde_with_macros",
"time 0.3.41",
"time",
]

[[package]]
@@ -13022,17 +13049,6 @@ dependencies = [
"weezl",
]

[[package]]
name = "time"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi 0.3.9",
]

[[package]]
name = "time"
version = "0.3.41"
@@ -13195,7 +13211,7 @@ dependencies = [
"derive_builder",
"esaxx-rs",
"getrandom 0.2.15",
"indicatif 0.17.11",
"indicatif",
"itertools 0.13.0",
"lazy_static",
"log",
@@ -14081,12 +14097,6 @@ dependencies = [
"try-lock",
]

[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"

[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@@ -15374,7 +15384,7 @@ dependencies = [
"oid-registry",
"rusticata-macros",
"thiserror 1.0.69",
"time 0.3.41",
"time",
]

[[package]]
@@ -15986,7 +15996,7 @@ dependencies = [
"rustls 0.23.25",
"rustls-webpki 0.102.8",
"serde",
"time 0.3.41",
"time",
"tokio",
"tokio-util",
"tracing",
@@ -16037,7 +16047,7 @@ dependencies = [
"rustls-pki-types",
"rustls-webpki 0.102.8",
"secrecy",
"time 0.3.41",
"time",
"tokio",
"tokio-util",
"tracing",
@@ -16122,7 +16132,7 @@ dependencies = [
"rustls-webpki 0.102.8",
"secrecy",
"socket2 0.5.8",
"time 0.3.41",
"time",
"tls-listener",
"tokio",
"tokio-rustls 0.26.2",
@@ -16608,29 +16618,44 @@ dependencies = [

[[package]]
name = "zip"
version = "0.5.13"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815"
checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164"
dependencies = [
"byteorder",
"arbitrary",
"crc32fast",
"crossbeam-utils",
"displaydoc",
"indexmap 2.8.0",
"num_enum",
"thiserror 1.0.69",
"time 0.1.45",
]

[[package]]
name = "zip"
version = "1.1.4"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164"
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
dependencies = [
"arbitrary",
"crc32fast",
"crossbeam-utils",
"displaydoc",
"indexmap 2.8.0",
"num_enum",
"thiserror 1.0.69",
"memchr",
"thiserror 2.0.12",
"time",
]

[[package]]
name = "zipsign-api"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dba6063ff82cdbd9a765add16d369abe81e520f836054e997c2db217ceca40c0"
dependencies = [
"base64 0.22.1",
"ed25519-dalek",
"thiserror 2.0.12",
]

[[package]]


+ 1
- 0
Cargo.toml View File

@@ -89,6 +89,7 @@ pyo3 = { version = "0.23", features = [
"multiple-pymethods",
] }
pythonize = "0.23"
git2 = { version = "0.18.0", features = ["vendored-openssl"] }

[package]
name = "dora-examples"


+ 2
- 2
binaries/cli/Cargo.toml View File

@@ -50,7 +50,7 @@ tabwriter = "1.4.0"
log = { version = "0.4.21", features = ["serde"] }
colored = "2.1.0"
env_logger = "0.11.3"
self_update = { version = "0.27.0", features = [
self_update = { version = "0.42.0", features = [
"rustls",
"archive-zip",
"archive-tar",
@@ -61,7 +61,7 @@ pyo3 = { workspace = true, features = [
"abi3",
], optional = true }
dunce = "1.0.5"
git2 = { version = "0.18.0", features = ["vendored-openssl"] }
git2 = { workspace = true }

[lib]
name = "dora_cli"


+ 6
- 19
binaries/cli/src/command/build/local.rs View File

@@ -71,27 +71,14 @@ async fn build_dataflow(
let mut info = BuildInfo {
node_working_dirs: Default::default(),
};
let mut errors = Vec::new();
for (node_id, task) in tasks {
match task.await {
Ok(node) => {
info.node_working_dirs
.insert(node_id, node.node_working_dir);
}
Err(err) => {
errors.push((node_id, err));
}
}
}
if errors.is_empty() {
Ok(info)
} else {
let mut message = "failed to build dataflow:\n".to_owned();
for (node_id, err) in errors {
message.push_str(&format!("- {node_id}: {err:?}\n-------------------\n\n"));
}
Err(eyre::eyre!(message))
let node = task
.await
.with_context(|| format!("failed to build node `{node_id}`"))?;
info.node_working_dirs
.insert(node_id, node.node_working_dir);
}
Ok(info)
}

struct LocalBuildLogger {


+ 1
- 1
binaries/daemon/Cargo.toml View File

@@ -45,6 +45,6 @@ crossbeam = "0.8.4"
crossbeam-skiplist = "0.1.3"
zenoh = "1.1.1"
url = "2.5.4"
git2 = { version = "0.18.0", features = ["vendored-openssl"] }
git2 = { workspace = true }
dunce = "1.0.5"
itertools = "0.14"

+ 6
- 19
binaries/daemon/src/lib.rs View File

@@ -946,32 +946,19 @@ impl Daemon {
let mut info = BuildInfo {
node_working_dirs: Default::default(),
};
let mut errors = Vec::new();
for task in tasks {
let NodeBuildTask {
node_id,
dynamic_node,
task,
} = task;
match task.await {
Ok(node) => {
info.node_working_dirs
.insert(node_id, node.node_working_dir);
}
Err(err) => {
errors.push((node_id, err));
}
}
}
if errors.is_empty() {
Ok(info)
} else {
let mut message = "failed to build dataflow:\n".to_owned();
for (node_id, err) in errors {
message.push_str(&format!("- {node_id}: {err:?}\n-------------------\n\n"));
}
Err(eyre::eyre!(message))
let node = task
.await
.with_context(|| format!("failed to build node `{node_id}`"))?;
info.node_working_dirs
.insert(node_id, node.node_working_dir);
}
Ok(info)
};

Ok(task)


+ 1
- 0
examples/rust-dataflow-git/.gitignore View File

@@ -1 +1,2 @@
/build
/git

+ 5
- 5
examples/rust-dataflow-git/dataflow.yml View File

@@ -1,9 +1,9 @@
nodes:
- id: rust-node
git: https://github.com/dora-rs/dora.git
rev: e31b2a34 # pinned commit, update this when changing the message crate
build: cargo build --manifest-path build/github.com/dora-rs/dora.git/e31b2a34/Cargo.toml -p rust-dataflow-example-node
path: build/github.com/dora-rs/dora.git/e31b2a34/target/debug/rust-dataflow-example-node
rev: a8c55b5a # pinned commit, update this when changing the message crate
build: cargo build -p rust-dataflow-example-node
path: target/debug/rust-dataflow-example-node
inputs:
tick: dora/timer/millis/10
outputs:
@@ -11,7 +11,7 @@ nodes:

- id: rust-status-node
git: https://github.com/dora-rs/dora.git
rev: e31b2a34 # pinned commit, update this when changing the message crate
rev: a8c55b5a # pinned commit, update this when changing the message crate
build: cargo build -p rust-dataflow-example-status-node
path: target/debug/rust-dataflow-example-status-node
inputs:
@@ -22,7 +22,7 @@ nodes:

- id: rust-sink
git: https://github.com/dora-rs/dora.git
rev: e31b2a34 # pinned commit, update this when changing the message crate
rev: a8c55b5a # pinned commit, update this when changing the message crate
build: cargo build -p rust-dataflow-example-sink
path: target/debug/rust-dataflow-example-sink
inputs:


+ 1
- 1
libraries/core/Cargo.toml View File

@@ -25,5 +25,5 @@ serde_json = "1.0.117"
log = { version = "0.4.21", features = ["serde"] }
dunce = "1.0.5"
url = "2.5.4"
git2 = { version = "0.18.0", features = ["vendored-openssl"] }
git2 = { workspace = true }
itertools = "0.14"

+ 53
- 28
libraries/core/src/build/git.rs View File

@@ -1,6 +1,6 @@
use crate::build::BuildLogger;
use dora_message::{common::LogLevel, descriptor::GitRepoRev, DataflowId, SessionId};
use eyre::{ContextCompat, WrapErr};
use eyre::{bail, ContextCompat, WrapErr};
use git2::FetchOptions;
use itertools::Itertools;
use std::{
@@ -145,15 +145,53 @@ impl GitFolder {
pub async fn prepare(self, logger: &mut impl BuildLogger) -> eyre::Result<PathBuf> {
let GitFolder { reuse } = self;

eprintln!("reuse: {reuse:?}");
let clone_dir = match reuse {
ReuseOptions::NewClone {
target_dir,
repo_url,
commit_hash,
} => {
let repository = clone_into(repo_url, &target_dir, logger).await?;
checkout_tree(&repository, &commit_hash)?;
target_dir
logger
.log_message(
LogLevel::Info,
format!(
"cloning {repo_url}#{commit_hash} into {}",
target_dir.display()
),
)
.await;
let clone_target = target_dir.clone();
let checkout_result = tokio::task::spawn_blocking(move || {
let repository = clone_into(repo_url.clone(), &clone_target)
.with_context(|| format!("failed to clone git repo from `{repo_url}`"))?;
checkout_tree(&repository, &commit_hash)
.with_context(|| format!("failed to checkout commit `{commit_hash}`"))
})
.await
.unwrap();

match checkout_result {
Ok(()) => target_dir,
Err(err) => {
logger
.log_message(LogLevel::Error, format!("{err:?}"))
.await;
// remove erroneous clone again
if let Err(err) = std::fs::remove_dir_all(target_dir) {
logger
.log_message(
LogLevel::Error,
format!(
"failed to remove clone dir after clone/checkout error: {}",
err.kind()
),
)
.await;
}
bail!(err)
}
}
}
ReuseOptions::CopyAndFetch {
from,
@@ -209,6 +247,7 @@ impl GitFolder {
}
}

#[derive(Debug)]
enum ReuseOptions {
/// Create a new clone of the repository.
NewClone {
@@ -241,35 +280,21 @@ fn rev_str(rev: &Option<GitRepoRev>) -> String {
}
}

async fn clone_into(
repo_addr: Url,
clone_dir: &Path,
logger: &mut impl BuildLogger,
) -> eyre::Result<git2::Repository> {
fn clone_into(repo_addr: Url, clone_dir: &Path) -> eyre::Result<git2::Repository> {
if let Some(parent) = clone_dir.parent() {
tokio::fs::create_dir_all(parent)
.await
std::fs::create_dir_all(parent)
.context("failed to create parent directory for git clone")?;
}

logger
.log_message(
LogLevel::Info,
format!("cloning {repo_addr} into {}", clone_dir.display()),
)
.await;
let clone_dir = clone_dir.to_owned();
let task = tokio::task::spawn_blocking(move || {
let mut builder = git2::build::RepoBuilder::new();
let mut fetch_options = git2::FetchOptions::new();
fetch_options.download_tags(git2::AutotagOption::All);
builder.fetch_options(fetch_options);
builder
.clone(repo_addr.as_str(), &clone_dir)
.context("failed to clone repo")
});
let repo = task.await??;
Ok(repo)

let mut builder = git2::build::RepoBuilder::new();
let mut fetch_options = git2::FetchOptions::new();
fetch_options.download_tags(git2::AutotagOption::All);
builder.fetch_options(fetch_options);
builder
.clone(repo_addr.as_str(), &clone_dir)
.context("failed to clone repo")
}

async fn fetch_changes(


Write Preview
Loading…
Cancel
Save