Browse Source

cargo fmt

pull/1072/head
Chrislearn Young Philipp Oppermann 5 months ago
parent
commit
26f9973ae3
Failed to extract signature
100 changed files with 272 additions and 243 deletions
  1. +2
    -3
      apis/c++/node/src/lib.rs
  2. +1
    -1
      apis/c++/operator/src/lib.rs
  3. +1
    -1
      apis/python/node/src/lib.rs
  4. +2
    -2
      apis/python/operator/src/lib.rs
  5. +2
    -2
      apis/rust/node/src/daemon_connection/mod.rs
  6. +12
    -13
      apis/rust/node/src/daemon_connection/tcp.rs
  7. +12
    -13
      apis/rust/node/src/daemon_connection/unix_domain.rs
  8. +4
    -4
      apis/rust/node/src/event_stream/mod.rs
  9. +1
    -1
      apis/rust/node/src/event_stream/thread.rs
  10. +3
    -3
      apis/rust/node/src/lib.rs
  11. +2
    -2
      apis/rust/node/src/node/control_channel.rs
  12. +2
    -2
      apis/rust/node/src/node/drop_stream.rs
  13. +6
    -4
      apis/rust/node/src/node/mod.rs
  14. +1
    -1
      apis/rust/operator/src/lib.rs
  15. +1
    -1
      apis/rust/operator/src/raw.rs
  16. +2
    -2
      binaries/cli/src/command/build/distributed.rs
  17. +2
    -2
      binaries/cli/src/command/build/mod.rs
  18. +3
    -3
      binaries/cli/src/command/check.rs
  19. +1
    -1
      binaries/cli/src/command/destroy.rs
  20. +2
    -2
      binaries/cli/src/command/list.rs
  21. +7
    -5
      binaries/cli/src/command/logs.rs
  22. +1
    -1
      binaries/cli/src/command/new.rs
  23. +1
    -1
      binaries/cli/src/command/run.rs
  24. +2
    -2
      binaries/cli/src/command/self_.rs
  25. +1
    -1
      binaries/cli/src/command/start/attach.rs
  26. +2
    -2
      binaries/cli/src/command/start/mod.rs
  27. +2
    -2
      binaries/cli/src/command/stop.rs
  28. +3
    -3
      binaries/cli/src/command/up.rs
  29. +2
    -2
      binaries/cli/src/common.rs
  30. +2
    -2
      binaries/cli/src/lib.rs
  31. +4
    -2
      binaries/cli/src/session.rs
  32. +1
    -1
      binaries/cli/src/template/c/mod.rs
  33. +1
    -1
      binaries/cli/src/template/cxx/mod.rs
  34. +1
    -1
      binaries/cli/src/template/python/mod.rs
  35. +1
    -1
      binaries/cli/src/template/rust/mod.rs
  36. +4
    -4
      binaries/coordinator/src/control.rs
  37. +21
    -9
      binaries/coordinator/src/lib.rs
  38. +2
    -2
      binaries/coordinator/src/listener.rs
  39. +3
    -3
      binaries/coordinator/src/run/mod.rs
  40. +6
    -4
      binaries/daemon/src/coordinator.rs
  41. +1
    -1
      binaries/daemon/src/local_listener.rs
  42. +1
    -1
      binaries/daemon/src/log.rs
  43. +5
    -5
      binaries/daemon/src/node_communication/mod.rs
  44. +2
    -2
      binaries/daemon/src/node_communication/tcp.rs
  45. +2
    -2
      binaries/daemon/src/node_communication/unix_domain.rs
  46. +4
    -4
      binaries/daemon/src/pending.rs
  47. +9
    -4
      binaries/runtime/src/lib.rs
  48. +1
    -1
      binaries/runtime/src/operator/channel.rs
  49. +8
    -10
      binaries/runtime/src/operator/python.rs
  50. +5
    -5
      binaries/runtime/src/operator/shared_lib.rs
  51. +1
    -1
      examples/benchmark/node/src/main.rs
  52. +1
    -1
      examples/benchmark/run.rs
  53. +4
    -2
      examples/c++-arrow-dataflow/run.rs
  54. +1
    -1
      examples/c++-dataflow/run.rs
  55. +1
    -1
      examples/c++-ros2-dataflow/run.rs
  56. +1
    -1
      examples/c-dataflow/run.rs
  57. +1
    -1
      examples/camera/run.rs
  58. +1
    -1
      examples/cmake-dataflow/run.rs
  59. +1
    -1
      examples/multiple-daemons/node/src/main.rs
  60. +1
    -1
      examples/multiple-daemons/operator/src/lib.rs
  61. +2
    -2
      examples/multiple-daemons/run.rs
  62. +4
    -2
      examples/multiple-daemons/sink/src/main.rs
  63. +1
    -1
      examples/python-dataflow/run.rs
  64. +1
    -1
      examples/python-operator-dataflow/run.rs
  65. +1
    -1
      examples/python-ros2-dataflow/run.rs
  66. +1
    -1
      examples/rerun-viewer/run.rs
  67. +1
    -1
      examples/rust-dataflow-git/run.rs
  68. +1
    -1
      examples/rust-dataflow-url/run.rs
  69. +1
    -1
      examples/rust-dataflow/node/src/main.rs
  70. +1
    -1
      examples/rust-dataflow/run.rs
  71. +5
    -3
      examples/rust-dataflow/sink-dynamic/src/main.rs
  72. +4
    -2
      examples/rust-dataflow/sink/src/main.rs
  73. +1
    -1
      examples/rust-dataflow/status-node/src/main.rs
  74. +3
    -4
      examples/rust-ros2-dataflow/node/src/main.rs
  75. +1
    -1
      examples/rust-ros2-dataflow/run.rs
  76. +1
    -1
      examples/vlm/run.rs
  77. +1
    -1
      libraries/arrow-convert/src/from_impls.rs
  78. +2
    -2
      libraries/arrow-convert/src/into_impls.rs
  79. +3
    -3
      libraries/arrow-convert/src/lib.rs
  80. +1
    -1
      libraries/communication-layer/pub-sub/src/zenoh.rs
  81. +8
    -8
      libraries/communication-layer/request-reply/src/lib.rs
  82. +16
    -16
      libraries/communication-layer/request-reply/src/tcp.rs
  83. +1
    -1
      libraries/core/src/build/build_command.rs
  84. +2
    -2
      libraries/core/src/build/git.rs
  85. +7
    -5
      libraries/core/src/descriptor/mod.rs
  86. +1
    -1
      libraries/core/src/descriptor/visualize.rs
  87. +1
    -1
      libraries/core/src/lib.rs
  88. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/parser/action.rs
  89. +2
    -2
      libraries/extensions/ros2-bridge/msg-gen/src/parser/constant.rs
  90. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/parser/ident.rs
  91. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/parser/literal.rs
  92. +2
    -2
      libraries/extensions/ros2-bridge/msg-gen/src/parser/member.rs
  93. +3
    -3
      libraries/extensions/ros2-bridge/msg-gen/src/parser/package.rs
  94. +2
    -2
      libraries/extensions/ros2-bridge/msg-gen/src/parser/types.rs
  95. +2
    -2
      libraries/extensions/ros2-bridge/msg-gen/src/types/action.rs
  96. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/types/constant.rs
  97. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/types/member.rs
  98. +2
    -2
      libraries/extensions/ros2-bridge/msg-gen/src/types/message.rs
  99. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/types/package.rs
  100. +1
    -1
      libraries/extensions/ros2-bridge/msg-gen/src/types/primitives.rs

+ 2
- 3
apis/c++/node/src/lib.rs View File

@@ -1,16 +1,15 @@
use std::{any::Any, vec}; use std::{any::Any, vec};


use dora_node_api::{ use dora_node_api::{
self,
self, Event, EventStream,
arrow::array::{AsArray, UInt8Array}, arrow::array::{AsArray, UInt8Array},
merged::{MergeExternal, MergedEvent}, merged::{MergeExternal, MergedEvent},
Event, EventStream,
}; };
use eyre::bail; use eyre::bail;


#[cfg(feature = "ros2-bridge")] #[cfg(feature = "ros2-bridge")]
use dora_ros2_bridge::{_core, ros2_client}; use dora_ros2_bridge::{_core, ros2_client};
use futures_lite::{stream, Stream, StreamExt};
use futures_lite::{Stream, StreamExt, stream};


#[cxx::bridge] #[cxx::bridge]
#[allow(clippy::needless_lifetimes)] #[allow(clippy::needless_lifetimes)]


+ 1
- 1
apis/c++/operator/src/lib.rs View File

@@ -2,7 +2,7 @@
#![warn(unsafe_op_in_unsafe_fn)] #![warn(unsafe_op_in_unsafe_fn)]


use dora_operator_api::{ use dora_operator_api::{
self, register_operator, DoraOperator, DoraOutputSender, DoraStatus, Event, IntoArrow,
self, DoraOperator, DoraOutputSender, DoraStatus, Event, IntoArrow, register_operator,
}; };
use ffi::DoraSendOutputResult; use ffi::DoraSendOutputResult;




+ 1
- 1
apis/python/node/src/lib.rs View File

@@ -11,7 +11,7 @@ use dora_node_api::dora_core::config::NodeId;
use dora_node_api::dora_core::descriptor::source_is_url; use dora_node_api::dora_core::descriptor::source_is_url;
use dora_node_api::merged::{MergeExternalSend, MergedEvent}; use dora_node_api::merged::{MergeExternalSend, MergedEvent};
use dora_node_api::{DataflowId, DoraNode, EventStream}; use dora_node_api::{DataflowId, DoraNode, EventStream};
use dora_operator_api_python::{pydict_to_metadata, DelayedCleanup, NodeCleanupHandle, PyEvent};
use dora_operator_api_python::{DelayedCleanup, NodeCleanupHandle, PyEvent, pydict_to_metadata};
use dora_ros2_bridge_python::Ros2Subscription; use dora_ros2_bridge_python::Ros2Subscription;
use eyre::Context; use eyre::Context;
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};


+ 2
- 2
apis/python/operator/src/lib.rs View File

@@ -5,8 +5,8 @@ use std::{


use arrow::pyarrow::ToPyArrow; use arrow::pyarrow::ToPyArrow;
use dora_node_api::{ use dora_node_api::{
merged::{MergeExternalSend, MergedEvent},
DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, StopCause, DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, StopCause,
merged::{MergeExternalSend, MergedEvent},
}; };
use eyre::{Context, Result}; use eyre::{Context, Result};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
@@ -282,7 +282,7 @@ mod tests {
use aligned_vec::{AVec, ConstAlign}; use aligned_vec::{AVec, ConstAlign};
use arrow::{ use arrow::{
array::{ array::{
ArrayData, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, Int8Array,
ArrayData, ArrayRef, BooleanArray, Float64Array, Int8Array, Int32Array, Int64Array,
ListArray, StructArray, ListArray, StructArray,
}, },
buffer::Buffer, buffer::Buffer,


+ 2
- 2
apis/rust/node/src/daemon_connection/mod.rs View File

@@ -1,10 +1,10 @@
use dora_core::{config::NodeId, uhlc::Timestamp}; use dora_core::{config::NodeId, uhlc::Timestamp};
use dora_message::{ use dora_message::{
DataflowId,
daemon_to_node::DaemonReply, daemon_to_node::DaemonReply,
node_to_daemon::{DaemonRequest, NodeRegisterRequest, Timestamped}, node_to_daemon::{DaemonRequest, NodeRegisterRequest, Timestamped},
DataflowId,
}; };
use eyre::{bail, eyre, Context};
use eyre::{Context, bail, eyre};
use shared_memory_server::{ShmemClient, ShmemConf}; use shared_memory_server::{ShmemClient, ShmemConf};
#[cfg(unix)] #[cfg(unix)]
use std::os::unix::net::UnixStream; use std::os::unix::net::UnixStream;


+ 12
- 13
apis/rust/node/src/daemon_connection/tcp.rs View File

@@ -2,7 +2,7 @@ use dora_message::{
daemon_to_node::DaemonReply, daemon_to_node::DaemonReply,
node_to_daemon::{DaemonRequest, Timestamped}, node_to_daemon::{DaemonRequest, Timestamped},
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use std::{ use std::{
io::{Read, Write}, io::{Read, Write},
net::TcpStream, net::TcpStream,
@@ -42,21 +42,20 @@ fn receive_reply(
connection: &mut TcpStream, connection: &mut TcpStream,
serializer: Serializer, serializer: Serializer,
) -> eyre::Result<Option<DaemonReply>> { ) -> eyre::Result<Option<DaemonReply>> {
let raw = match tcp_receive(connection) {
Ok(raw) => raw,
Err(err) => match err.kind() {
std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
return Ok(None)
}
other => {
return Err(err).with_context(|| {
let raw =
match tcp_receive(connection) {
Ok(raw) => raw,
Err(err) => match err.kind() {
std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
return Ok(None);
}
other => return Err(err).with_context(|| {
format!( format!(
"unexpected I/O error (kind {other:?}) while trying to receive DaemonReply" "unexpected I/O error (kind {other:?}) while trying to receive DaemonReply"
) )
})
}
},
};
}),
},
};
match serializer { match serializer {
Serializer::Bincode => bincode::deserialize(&raw) Serializer::Bincode => bincode::deserialize(&raw)
.wrap_err("failed to deserialize DaemonReply") .wrap_err("failed to deserialize DaemonReply")


+ 12
- 13
apis/rust/node/src/daemon_connection/unix_domain.rs View File

@@ -2,7 +2,7 @@ use dora_message::{
daemon_to_node::DaemonReply, daemon_to_node::DaemonReply,
node_to_daemon::{DaemonRequest, Timestamped}, node_to_daemon::{DaemonRequest, Timestamped},
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use std::{ use std::{
io::{Read, Write}, io::{Read, Write},
os::unix::net::UnixStream, os::unix::net::UnixStream,
@@ -42,21 +42,20 @@ fn receive_reply(
connection: &mut UnixStream, connection: &mut UnixStream,
serializer: Serializer, serializer: Serializer,
) -> eyre::Result<Option<DaemonReply>> { ) -> eyre::Result<Option<DaemonReply>> {
let raw = match stream_receive(connection) {
Ok(raw) => raw,
Err(err) => match err.kind() {
std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
return Ok(None)
}
other => {
return Err(err).with_context(|| {
let raw =
match stream_receive(connection) {
Ok(raw) => raw,
Err(err) => match err.kind() {
std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionAborted => {
return Ok(None);
}
other => return Err(err).with_context(|| {
format!( format!(
"unexpected I/O error (kind {other:?}) while trying to receive DaemonReply" "unexpected I/O error (kind {other:?}) while trying to receive DaemonReply"
) )
})
}
},
};
}),
},
};
match serializer { match serializer {
Serializer::Bincode => bincode::deserialize(&raw) Serializer::Bincode => bincode::deserialize(&raw)
.wrap_err("failed to deserialize DaemonReply") .wrap_err("failed to deserialize DaemonReply")


+ 4
- 4
apis/rust/node/src/event_stream/mod.rs View File

@@ -6,18 +6,18 @@ use std::{
}; };


use dora_message::{ use dora_message::{
DataflowId,
daemon_to_node::{DaemonCommunication, DaemonReply, DataMessage, NodeEvent}, daemon_to_node::{DaemonCommunication, DaemonReply, DataMessage, NodeEvent},
id::DataId, id::DataId,
node_to_daemon::{DaemonRequest, Timestamped}, node_to_daemon::{DaemonRequest, Timestamped},
DataflowId,
}; };
pub use event::{Event, StopCause}; pub use event::{Event, StopCause};
use futures::{ use futures::{
future::{select, Either},
Stream, StreamExt, Stream, StreamExt,
future::{Either, select},
}; };
use futures_timer::Delay; use futures_timer::Delay;
use scheduler::{Scheduler, NON_INPUT_EVENT};
use scheduler::{NON_INPUT_EVENT, Scheduler};


use self::thread::{EventItem, EventStreamThreadHandle}; use self::thread::{EventItem, EventStreamThreadHandle};
use crate::{ use crate::{
@@ -28,7 +28,7 @@ use dora_core::{
config::{Input, NodeId}, config::{Input, NodeId},
uhlc, uhlc,
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};


pub use scheduler::Scheduler as EventScheduler; pub use scheduler::Scheduler as EventScheduler;




+ 1
- 1
apis/rust/node/src/event_stream/thread.rs View File

@@ -6,7 +6,7 @@ use dora_message::{
daemon_to_node::{DaemonReply, NodeEvent}, daemon_to_node::{DaemonReply, NodeEvent},
node_to_daemon::{DaemonRequest, DropToken, Timestamped}, node_to_daemon::{DaemonRequest, DropToken, Timestamped},
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use flume::RecvTimeoutError; use flume::RecvTimeoutError;
use std::{ use std::{
sync::Arc, sync::Arc,


+ 3
- 3
apis/rust/node/src/lib.rs View File

@@ -84,13 +84,13 @@ pub use arrow;
pub use dora_arrow_convert::*; pub use dora_arrow_convert::*;
pub use dora_core::{self, uhlc}; pub use dora_core::{self, uhlc};
pub use dora_message::{ pub use dora_message::{
metadata::{Metadata, MetadataParameters, Parameter},
DataflowId, DataflowId,
metadata::{Metadata, MetadataParameters, Parameter},
}; };
pub use event_stream::{merged, Event, EventScheduler, EventStream, StopCause};
pub use event_stream::{Event, EventScheduler, EventStream, StopCause, merged};
pub use flume::Receiver; pub use flume::Receiver;
pub use futures; pub use futures;
pub use node::{arrow_utils, DataSample, DoraNode, ZERO_COPY_THRESHOLD};
pub use node::{DataSample, DoraNode, ZERO_COPY_THRESHOLD, arrow_utils};


mod daemon_connection; mod daemon_connection;
mod event_stream; mod event_stream;


+ 2
- 2
apis/rust/node/src/node/control_channel.rs View File

@@ -6,12 +6,12 @@ use dora_core::{
uhlc::HLC, uhlc::HLC,
}; };
use dora_message::{ use dora_message::{
DataflowId,
daemon_to_node::{DaemonCommunication, DaemonReply}, daemon_to_node::{DaemonCommunication, DaemonReply},
metadata::Metadata, metadata::Metadata,
node_to_daemon::{DaemonRequest, DataMessage, Timestamped}, node_to_daemon::{DaemonRequest, DataMessage, Timestamped},
DataflowId,
}; };
use eyre::{bail, eyre, Context};
use eyre::{Context, bail, eyre};


pub(crate) struct ControlChannel { pub(crate) struct ControlChannel {
channel: DaemonChannel, channel: DaemonChannel,


+ 2
- 2
apis/rust/node/src/node/drop_stream.rs View File

@@ -3,11 +3,11 @@ use std::{sync::Arc, time::Duration};
use crate::daemon_connection::DaemonChannel; use crate::daemon_connection::DaemonChannel;
use dora_core::{config::NodeId, uhlc}; use dora_core::{config::NodeId, uhlc};
use dora_message::{ use dora_message::{
DataflowId,
daemon_to_node::{DaemonCommunication, DaemonReply, NodeDropEvent}, daemon_to_node::{DaemonCommunication, DaemonReply, NodeDropEvent},
node_to_daemon::{DaemonRequest, DropToken, Timestamped}, node_to_daemon::{DaemonRequest, DropToken, Timestamped},
DataflowId,
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use flume::RecvTimeoutError; use flume::RecvTimeoutError;


pub struct DropStream { pub struct DropStream {


+ 6
- 4
apis/rust/node/src/node/mod.rs View File

@@ -1,4 +1,4 @@
use crate::{daemon_connection::DaemonChannel, EventStream};
use crate::{EventStream, daemon_connection::DaemonChannel};


use self::{ use self::{
arrow_utils::{copy_array_into_sample, required_data_size}, arrow_utils::{copy_array_into_sample, required_data_size},
@@ -16,12 +16,12 @@ use dora_core::{
}; };


use dora_message::{ use dora_message::{
DataflowId,
daemon_to_node::{DaemonReply, NodeConfig}, daemon_to_node::{DaemonReply, NodeConfig},
metadata::{ArrowTypeInfo, Metadata, MetadataParameters}, metadata::{ArrowTypeInfo, Metadata, MetadataParameters},
node_to_daemon::{DaemonRequest, DataMessage, DropToken, Timestamped}, node_to_daemon::{DaemonRequest, DataMessage, DropToken, Timestamped},
DataflowId,
}; };
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use shared_memory_extended::{Shmem, ShmemConf}; use shared_memory_extended::{Shmem, ShmemConf};
use std::{ use std::{
collections::{BTreeSet, HashMap, VecDeque}, collections::{BTreeSet, HashMap, VecDeque},
@@ -157,7 +157,9 @@ impl DoraNode {
/// [`init_from_node_id`][Self::init_from_node_id]. /// [`init_from_node_id`][Self::init_from_node_id].
pub fn init_flexible(node_id: NodeId) -> eyre::Result<(Self, EventStream)> { pub fn init_flexible(node_id: NodeId) -> eyre::Result<(Self, EventStream)> {
if std::env::var("DORA_NODE_CONFIG").is_ok() { if std::env::var("DORA_NODE_CONFIG").is_ok() {
info!("Skipping {node_id} specified within the node initialization in favor of `DORA_NODE_CONFIG` specified by `dora start`");
info!(
"Skipping {node_id} specified within the node initialization in favor of `DORA_NODE_CONFIG` specified by `dora start`"
);
Self::init_from_env() Self::init_from_env()
} else { } else {
Self::init_from_node_id(node_id) Self::init_from_node_id(node_id)


+ 1
- 1
apis/rust/operator/src/lib.rs View File

@@ -23,8 +23,8 @@ pub use dora_operator_api_macros::register_operator;
pub use dora_operator_api_types as types; pub use dora_operator_api_types as types;
pub use types::DoraStatus; pub use types::DoraStatus;
use types::{ use types::{
arrow::{self, array::Array},
Metadata, Output, SendOutput, Metadata, Output, SendOutput,
arrow::{self, array::Array},
}; };


pub mod raw; pub mod raw;


+ 1
- 1
apis/rust/operator/src/raw.rs View File

@@ -1,6 +1,6 @@
use crate::{DoraOperator, DoraOutputSender, DoraStatus, Event}; use crate::{DoraOperator, DoraOutputSender, DoraStatus, Event};
use dora_operator_api_types::{ use dora_operator_api_types::{
arrow, DoraInitResult, DoraResult, OnEventResult, RawEvent, SendOutput,
DoraInitResult, DoraResult, OnEventResult, RawEvent, SendOutput, arrow,
}; };
use std::ffi::c_void; use std::ffi::c_void;




+ 2
- 2
binaries/cli/src/command/build/distributed.rs View File

@@ -1,13 +1,13 @@
use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection}; use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
use dora_core::descriptor::Descriptor; use dora_core::descriptor::Descriptor;
use dora_message::{ use dora_message::{
BuildId,
cli_to_coordinator::ControlRequest, cli_to_coordinator::ControlRequest,
common::{GitSource, LogMessage}, common::{GitSource, LogMessage},
coordinator_to_cli::ControlRequestReply, coordinator_to_cli::ControlRequestReply,
id::NodeId, id::NodeId,
BuildId,
}; };
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
collections::BTreeMap, collections::BTreeMap,
net::{SocketAddr, TcpStream}, net::{SocketAddr, TcpStream},


+ 2
- 2
binaries/cli/src/command/build/mod.rs View File

@@ -52,11 +52,11 @@ use dora_core::{
descriptor::{CoreNodeKind, CustomNode, Descriptor, DescriptorExt}, descriptor::{CoreNodeKind, CustomNode, Descriptor, DescriptorExt},
topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST}, topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST},
}; };
use dora_message::{descriptor::NodeSource, BuildId};
use dora_message::{BuildId, descriptor::NodeSource};
use eyre::Context; use eyre::Context;
use std::{collections::BTreeMap, net::IpAddr}; use std::{collections::BTreeMap, net::IpAddr};


use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use crate::{ use crate::{
common::{connect_to_coordinator, local_working_dir, resolve_dataflow}, common::{connect_to_coordinator, local_working_dir, resolve_dataflow},
session::DataflowSession, session::DataflowSession,


+ 3
- 3
binaries/cli/src/command/check.rs View File

@@ -1,10 +1,10 @@
use super::{default_tracing, Executable};
use crate::{common::connect_to_coordinator, LOCALHOST};
use super::{Executable, default_tracing};
use crate::{LOCALHOST, common::connect_to_coordinator};
use communication_layer_request_reply::TcpRequestReplyConnection; use communication_layer_request_reply::TcpRequestReplyConnection;
use dora_core::descriptor::DescriptorExt; use dora_core::descriptor::DescriptorExt;
use dora_core::{descriptor::Descriptor, topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT}; use dora_core::{descriptor::Descriptor, topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT};
use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply}; use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
io::{IsTerminal, Write}, io::{IsTerminal, Write},
net::SocketAddr, net::SocketAddr,


+ 1
- 1
binaries/cli/src/command/destroy.rs View File

@@ -1,4 +1,4 @@
use super::{default_tracing, up, Executable};
use super::{Executable, default_tracing, up};
use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST}; use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST};
use std::net::IpAddr; use std::net::IpAddr;
use std::path::PathBuf; use std::path::PathBuf;


+ 2
- 2
binaries/cli/src/command/list.rs View File

@@ -1,9 +1,9 @@
use std::io::Write; use std::io::Write;


use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use crate::{ use crate::{
common::{connect_to_coordinator, query_running_dataflows},
LOCALHOST, LOCALHOST,
common::{connect_to_coordinator, query_running_dataflows},
}; };
use clap::Args; use clap::Args;
use communication_layer_request_reply::TcpRequestReplyConnection; use communication_layer_request_reply::TcpRequestReplyConnection;


+ 7
- 5
binaries/cli/src/command/logs.rs View File

@@ -1,11 +1,11 @@
use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use crate::common::{connect_to_coordinator, query_running_dataflows}; use crate::common::{connect_to_coordinator, query_running_dataflows};
use bat::{Input, PrettyPrinter}; use bat::{Input, PrettyPrinter};
use clap::Args; use clap::Args;
use communication_layer_request_reply::TcpRequestReplyConnection; use communication_layer_request_reply::TcpRequestReplyConnection;
use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST}; use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST};
use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply}; use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
use eyre::{bail, Context, Result};
use eyre::{Context, Result, bail};
use uuid::Uuid; use uuid::Uuid;


#[derive(Debug, Args)] #[derive(Debug, Args)]
@@ -80,9 +80,11 @@ pub fn logs(
.grid(false) .grid(false)
.line_numbers(false) .line_numbers(false)
.paging_mode(bat::PagingMode::QuitIfOneScreen) .paging_mode(bat::PagingMode::QuitIfOneScreen)
.inputs(vec![Input::from_bytes(&logs)
.name("Logs")
.title(format!("Logs from {node}.").as_str())])
.inputs(vec![
Input::from_bytes(&logs)
.name("Logs")
.title(format!("Logs from {node}.").as_str()),
])
.print() .print()
.wrap_err("Something went wrong with viewing log file")?; .wrap_err("Something went wrong with viewing log file")?;




+ 1
- 1
binaries/cli/src/command/new.rs View File

@@ -1,6 +1,6 @@
use clap::Args; use clap::Args;


use super::{default_tracing, Executable};
use super::{Executable, default_tracing};


#[derive(Debug, Args)] #[derive(Debug, Args)]
/// Generate a new project or node. Choose the language between Rust, Python, C or C++. /// Generate a new project or node. Choose the language between Rust, Python, C or C++.


+ 1
- 1
binaries/cli/src/command/run.rs View File

@@ -11,7 +11,7 @@ use crate::{
output::print_log_message, output::print_log_message,
session::DataflowSession, session::DataflowSession,
}; };
use dora_daemon::{flume, Daemon, LogDestination};
use dora_daemon::{Daemon, LogDestination, flume};
use dora_tracing::TracingBuilder; use dora_tracing::TracingBuilder;
use eyre::Context; use eyre::Context;
use tokio::runtime::Builder; use tokio::runtime::Builder;


+ 2
- 2
binaries/cli/src/command/self_.rs View File

@@ -1,6 +1,6 @@
use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use clap::Subcommand; use clap::Subcommand;
use eyre::{bail, Context};
use eyre::{Context, bail};


#[derive(Debug, Subcommand)] #[derive(Debug, Subcommand)]
/// Dora CLI self-management commands /// Dora CLI self-management commands


+ 1
- 1
binaries/cli/src/command/start/attach.rs View File

@@ -1,5 +1,5 @@
use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection}; use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
use dora_core::descriptor::{resolve_path, CoreNodeKind, Descriptor, DescriptorExt};
use dora_core::descriptor::{CoreNodeKind, Descriptor, DescriptorExt, resolve_path};
use dora_message::cli_to_coordinator::ControlRequest; use dora_message::cli_to_coordinator::ControlRequest;
use dora_message::common::LogMessage; use dora_message::common::LogMessage;
use dora_message::coordinator_to_cli::ControlRequestReply; use dora_message::coordinator_to_cli::ControlRequestReply;


+ 2
- 2
binaries/cli/src/command/start/mod.rs View File

@@ -2,7 +2,7 @@
//! //!
//! The `dora start` command does not run any build commands, nor update git dependencies or similar. Use `dora build` for that. //! The `dora start` command does not run any build commands, nor update git dependencies or similar. Use `dora build` for that.


use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use crate::{ use crate::{
command::start::attach::attach_dataflow, command::start::attach::attach_dataflow,
common::{connect_to_coordinator, local_working_dir, resolve_dataflow}, common::{connect_to_coordinator, local_working_dir, resolve_dataflow},
@@ -17,7 +17,7 @@ use dora_core::{
use dora_message::{ use dora_message::{
cli_to_coordinator::ControlRequest, common::LogMessage, coordinator_to_cli::ControlRequestReply, cli_to_coordinator::ControlRequest, common::LogMessage, coordinator_to_cli::ControlRequestReply,
}; };
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
net::{IpAddr, SocketAddr, TcpStream}, net::{IpAddr, SocketAddr, TcpStream},
path::PathBuf, path::PathBuf,


+ 2
- 2
binaries/cli/src/command/stop.rs View File

@@ -1,11 +1,11 @@
use super::{default_tracing, Executable};
use super::{Executable, default_tracing};
use crate::common::{connect_to_coordinator, handle_dataflow_result, query_running_dataflows}; use crate::common::{connect_to_coordinator, handle_dataflow_result, query_running_dataflows};
use communication_layer_request_reply::TcpRequestReplyConnection; use communication_layer_request_reply::TcpRequestReplyConnection;
use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST}; use dora_core::topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, LOCALHOST};
use dora_message::cli_to_coordinator::ControlRequest; use dora_message::cli_to_coordinator::ControlRequest;
use dora_message::coordinator_to_cli::ControlRequestReply; use dora_message::coordinator_to_cli::ControlRequestReply;
use duration_str::parse; use duration_str::parse;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::net::IpAddr; use std::net::IpAddr;
use std::time::Duration; use std::time::Duration;
use uuid::Uuid; use uuid::Uuid;


+ 3
- 3
binaries/cli/src/command/up.rs View File

@@ -1,9 +1,9 @@
use super::check::daemon_running; use super::check::daemon_running;
use super::{default_tracing, Executable};
use crate::{common::connect_to_coordinator, LOCALHOST};
use super::{Executable, default_tracing};
use crate::{LOCALHOST, common::connect_to_coordinator};
use dora_core::topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT; use dora_core::topics::DORA_COORDINATOR_PORT_CONTROL_DEFAULT;
use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply}; use dora_message::{cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply};
use eyre::{bail, Context, ContextCompat};
use eyre::{Context, ContextCompat, bail};
use std::path::PathBuf; use std::path::PathBuf;
use std::{fs, net::SocketAddr, path::Path, process::Command, time::Duration}; use std::{fs, net::SocketAddr, path::Path, process::Command, time::Duration};




+ 2
- 2
binaries/cli/src/common.rs View File

@@ -1,12 +1,12 @@
use crate::formatting::FormatDataflowError; use crate::formatting::FormatDataflowError;
use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection}; use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection};
use dora_core::descriptor::{source_is_url, Descriptor};
use dora_core::descriptor::{Descriptor, source_is_url};
use dora_download::download_file; use dora_download::download_file;
use dora_message::{ use dora_message::{
cli_to_coordinator::ControlRequest, cli_to_coordinator::ControlRequest,
coordinator_to_cli::{ControlRequestReply, DataflowList, DataflowResult}, coordinator_to_cli::{ControlRequestReply, DataflowList, DataflowResult},
}; };
use eyre::{bail, Context, ContextCompat};
use eyre::{Context, ContextCompat, bail};
use std::{ use std::{
env::current_dir, env::current_dir,
net::SocketAddr, net::SocketAddr,


+ 2
- 2
binaries/cli/src/lib.rs View File

@@ -65,9 +65,9 @@ pub fn lib_main(args: Args) {
use clap::Parser; use clap::Parser;
#[cfg(feature = "python")] #[cfg(feature = "python")]
use pyo3::{ use pyo3::{
pyfunction, pymodule,
Bound, PyResult, Python, pyfunction, pymodule,
types::{PyModule, PyModuleMethods}, types::{PyModule, PyModuleMethods},
wrap_pyfunction, Bound, PyResult, Python,
wrap_pyfunction,
}; };


#[cfg(feature = "python")] #[cfg(feature = "python")]


+ 4
- 2
binaries/cli/src/session.rs View File

@@ -4,7 +4,7 @@ use std::{
}; };


use dora_core::build::BuildInfo; use dora_core::build::BuildInfo;
use dora_message::{common::GitSource, id::NodeId, BuildId, SessionId};
use dora_message::{BuildId, SessionId, common::GitSource, id::NodeId};
use eyre::{Context, ContextCompat}; use eyre::{Context, ContextCompat};


#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -33,7 +33,9 @@ impl DataflowSession {
if let Ok(parsed) = deserialize(&session_file) { if let Ok(parsed) = deserialize(&session_file) {
return Ok(parsed); return Ok(parsed);
} else { } else {
tracing::warn!("failed to read dataflow session file, regenerating (you might need to run `dora build` again)");
tracing::warn!(
"failed to read dataflow session file, regenerating (you might need to run `dora build` again)"
);
} }
} }




+ 1
- 1
binaries/cli/src/template/c/mod.rs View File

@@ -1,5 +1,5 @@
use dora_node_api_c::HEADER_NODE_API; use dora_node_api_c::HEADER_NODE_API;
use eyre::{bail, Context, ContextCompat};
use eyre::{Context, ContextCompat, bail};
use std::{ use std::{
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},


+ 1
- 1
binaries/cli/src/template/cxx/mod.rs View File

@@ -1,4 +1,4 @@
use eyre::{bail, Context, ContextCompat};
use eyre::{Context, ContextCompat, bail};
use std::{ use std::{
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},


+ 1
- 1
binaries/cli/src/template/python/mod.rs View File

@@ -1,4 +1,4 @@
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},


+ 1
- 1
binaries/cli/src/template/rust/mod.rs View File

@@ -1,4 +1,4 @@
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},


+ 4
- 4
binaries/coordinator/src/control.rs View File

@@ -1,15 +1,15 @@
use crate::{ use crate::{
tcp_utils::{tcp_receive, tcp_send},
Event, Event,
tcp_utils::{tcp_receive, tcp_send},
}; };
use dora_message::{ use dora_message::{
cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply, BuildId,
BuildId, cli_to_coordinator::ControlRequest, coordinator_to_cli::ControlRequestReply,
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use futures::{ use futures::{
FutureExt, Stream, StreamExt,
future::{self, Either}, future::{self, Either},
stream::FuturesUnordered, stream::FuturesUnordered,
FutureExt, Stream, StreamExt,
}; };
use futures_concurrency::future::Race; use futures_concurrency::future::Race;
use std::{io::ErrorKind, net::SocketAddr}; use std::{io::ErrorKind, net::SocketAddr};


+ 21
- 9
binaries/coordinator/src/lib.rs View File

@@ -9,6 +9,7 @@ use dora_core::{
uhlc::{self, HLC}, uhlc::{self, HLC},
}; };
use dora_message::{ use dora_message::{
BuildId, DataflowId, SessionId,
cli_to_coordinator::ControlRequest, cli_to_coordinator::ControlRequest,
common::{DaemonId, GitSource}, common::{DaemonId, GitSource},
coordinator_to_cli::{ coordinator_to_cli::{
@@ -20,10 +21,9 @@ use dora_message::{
}, },
daemon_to_coordinator::{DaemonCoordinatorReply, DataflowDaemonResult}, daemon_to_coordinator::{DaemonCoordinatorReply, DataflowDaemonResult},
descriptor::{Descriptor, ResolvedNode}, descriptor::{Descriptor, ResolvedNode},
BuildId, DataflowId, SessionId,
}; };
use eyre::{bail, eyre, ContextCompat, Result, WrapErr};
use futures::{future::join_all, stream::FuturesUnordered, Future, Stream, StreamExt};
use eyre::{ContextCompat, Result, WrapErr, bail, eyre};
use futures::{Future, Stream, StreamExt, future::join_all, stream::FuturesUnordered};
use futures_concurrency::stream::Merge; use futures_concurrency::stream::Merge;
use itertools::Itertools; use itertools::Itertools;
use log_subscriber::LogSubscriber; use log_subscriber::LogSubscriber;
@@ -122,7 +122,9 @@ fn resolve_name(
Ok(*uuid) Ok(*uuid)
} else { } else {
// TODO: Index the archived dataflows in order to return logs based on the index. // TODO: Index the archived dataflows in order to return logs based on the index.
bail!("multiple archived dataflows found with name `{name}`, Please provide the UUID instead.");
bail!(
"multiple archived dataflows found with name `{name}`, Please provide the UUID instead."
);
} }
} else if let [uuid] = uuids.as_slice() { } else if let [uuid] = uuids.as_slice() {
Ok(*uuid) Ok(*uuid)
@@ -285,7 +287,9 @@ async fn start_inner(
); );
} }
Err(err) => { Err(err) => {
tracing::warn!("failed to register daemon connection for daemon `{daemon_id}`: {err}");
tracing::warn!(
"failed to register daemon connection for daemon `{daemon_id}`: {err}"
);
} }
} }
} }
@@ -341,7 +345,9 @@ async fn start_inner(
} }
} }
DataflowEvent::DataflowFinishedOnDaemon { daemon_id, result } => { DataflowEvent::DataflowFinishedOnDaemon { daemon_id, result } => {
tracing::debug!("coordinator received DataflowFinishedOnDaemon ({daemon_id:?}, result: {result:?})");
tracing::debug!(
"coordinator received DataflowFinishedOnDaemon ({daemon_id:?}, result: {result:?})"
);
match running_dataflows.entry(uuid) { match running_dataflows.entry(uuid) {
std::collections::hash_map::Entry::Occupied(mut entry) => { std::collections::hash_map::Entry::Occupied(mut entry) => {
let dataflow = entry.get_mut(); let dataflow = entry.get_mut();
@@ -474,7 +480,9 @@ async fn start_inner(
.values() .values()
.any(|d: &RunningDataflow| d.name.as_deref() == Some(name)) .any(|d: &RunningDataflow| d.name.as_deref() == Some(name))
{ {
bail!("there is already a running dataflow with name `{name}`");
bail!(
"there is already a running dataflow with name `{name}`"
);
} }
} }
let dataflow = start_dataflow( let dataflow = start_dataflow(
@@ -875,7 +883,9 @@ async fn start_inner(
} }
} }
None => { None => {
tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
tracing::warn!(
"received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map"
);
} }
}, },
Event::DataflowSpawnResult { Event::DataflowSpawnResult {
@@ -901,7 +911,9 @@ async fn start_inner(
}; };
} }
None => { None => {
tracing::warn!("received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map");
tracing::warn!(
"received DataflowSpawnResult, but no matching dataflow in `running_dataflows` map"
);
} }
}, },
} }


+ 2
- 2
binaries/coordinator/src/listener.rs View File

@@ -1,4 +1,4 @@
use crate::{tcp_utils::tcp_receive, DaemonRequest, DataflowEvent, Event};
use crate::{DaemonRequest, DataflowEvent, Event, tcp_utils::tcp_receive};
use dora_core::uhlc::HLC; use dora_core::uhlc::HLC;
use dora_message::daemon_to_coordinator::{CoordinatorRequest, DaemonEvent, Timestamped}; use dora_message::daemon_to_coordinator::{CoordinatorRequest, DaemonEvent, Timestamped};
use eyre::Context; use eyre::Context;
@@ -12,7 +12,7 @@ pub async fn create_listener(bind: SocketAddr) -> eyre::Result<TcpListener> {
let socket = match TcpListener::bind(bind).await { let socket = match TcpListener::bind(bind).await {
Ok(socket) => socket, Ok(socket) => socket,
Err(err) => { Err(err) => {
return Err(eyre::Report::new(err).wrap_err("failed to create local TCP listener"))
return Err(eyre::Report::new(err).wrap_err("failed to create local TCP listener"));
} }
}; };
Ok(socket) Ok(socket)


+ 3
- 3
binaries/coordinator/src/run/mod.rs View File

@@ -1,18 +1,18 @@
use crate::{ use crate::{
tcp_utils::{tcp_receive, tcp_send},
DaemonConnections, DaemonConnections,
tcp_utils::{tcp_receive, tcp_send},
}; };


use dora_core::{descriptor::DescriptorExt, uhlc::HLC}; use dora_core::{descriptor::DescriptorExt, uhlc::HLC};
use dora_message::{ use dora_message::{
BuildId, SessionId,
common::DaemonId, common::DaemonId,
coordinator_to_daemon::{DaemonCoordinatorEvent, SpawnDataflowNodes, Timestamped}, coordinator_to_daemon::{DaemonCoordinatorEvent, SpawnDataflowNodes, Timestamped},
daemon_to_coordinator::DaemonCoordinatorReply, daemon_to_coordinator::DaemonCoordinatorReply,
descriptor::{Descriptor, ResolvedNode}, descriptor::{Descriptor, ResolvedNode},
id::NodeId, id::NodeId,
BuildId, SessionId,
}; };
use eyre::{bail, eyre, ContextCompat, WrapErr};
use eyre::{ContextCompat, WrapErr, bail, eyre};
use itertools::Itertools; use itertools::Itertools;
use std::{ use std::{
collections::{BTreeMap, BTreeSet}, collections::{BTreeMap, BTreeSet},


+ 6
- 4
binaries/daemon/src/coordinator.rs View File

@@ -1,6 +1,6 @@
use crate::{ use crate::{
socket_stream_utils::{socket_stream_receive, socket_stream_send},
DaemonCoordinatorEvent, DaemonCoordinatorEvent,
socket_stream_utils::{socket_stream_receive, socket_stream_send},
}; };
use dora_core::uhlc::HLC; use dora_core::uhlc::HLC;
use dora_message::{ use dora_message::{
@@ -8,14 +8,14 @@ use dora_message::{
coordinator_to_daemon::RegisterResult, coordinator_to_daemon::RegisterResult,
daemon_to_coordinator::{CoordinatorRequest, DaemonCoordinatorReply, DaemonRegisterRequest}, daemon_to_coordinator::{CoordinatorRequest, DaemonCoordinatorReply, DaemonRegisterRequest},
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use std::{io::ErrorKind, net::SocketAddr, time::Duration}; use std::{io::ErrorKind, net::SocketAddr, time::Duration};
use tokio::{ use tokio::{
net::TcpStream, net::TcpStream,
sync::{mpsc, oneshot}, sync::{mpsc, oneshot},
time::sleep, time::sleep,
}; };
use tokio_stream::{wrappers::ReceiverStream, Stream};
use tokio_stream::{Stream, wrappers::ReceiverStream};
use tracing::warn; use tracing::warn;


const DAEMON_COORDINATOR_RETRY_INTERVAL: std::time::Duration = Duration::from_secs(1); const DAEMON_COORDINATOR_RETRY_INTERVAL: std::time::Duration = Duration::from_secs(1);
@@ -37,7 +37,9 @@ pub async fn register(
.wrap_err("failed to connect to dora-coordinator") .wrap_err("failed to connect to dora-coordinator")
{ {
Err(err) => { Err(err) => {
warn!("Could not connect to: {addr}, with error: {err}. Retring in {DAEMON_COORDINATOR_RETRY_INTERVAL:#?}..");
warn!(
"Could not connect to: {addr}, with error: {err}. Retring in {DAEMON_COORDINATOR_RETRY_INTERVAL:#?}.."
);
sleep(DAEMON_COORDINATOR_RETRY_INTERVAL).await; sleep(DAEMON_COORDINATOR_RETRY_INTERVAL).await;
} }
Ok(stream) => { Ok(stream) => {


+ 1
- 1
binaries/daemon/src/local_listener.rs View File

@@ -137,7 +137,7 @@ async fn receive_message(
| ErrorKind::ConnectionReset => return Ok(None), | ErrorKind::ConnectionReset => return Ok(None),
_other => { _other => {
return Err(err) return Err(err)
.context("unexpected I/O error while trying to receive DaemonRequest")
.context("unexpected I/O error while trying to receive DaemonRequest");
} }
}, },
}; };


+ 1
- 1
binaries/daemon/src/log.rs View File

@@ -10,9 +10,9 @@ use dora_core::{
uhlc, uhlc,
}; };
use dora_message::{ use dora_message::{
BuildId,
common::{DaemonId, LogLevel, LogMessage, Timestamped}, common::{DaemonId, LogLevel, LogMessage, Timestamped},
daemon_to_coordinator::{CoordinatorRequest, DaemonEvent}, daemon_to_coordinator::{CoordinatorRequest, DaemonEvent},
BuildId,
}; };
use eyre::Context; use eyre::Context;
use flume::Sender; use flume::Sender;


+ 5
- 5
binaries/daemon/src/node_communication/mod.rs View File

@@ -5,13 +5,13 @@ use dora_core::{
uhlc, uhlc,
}; };
use dora_message::{ use dora_message::{
DataflowId,
common::{DropToken, Timestamped}, common::{DropToken, Timestamped},
daemon_to_node::{DaemonCommunication, DaemonReply, NodeDropEvent, NodeEvent}, daemon_to_node::{DaemonCommunication, DaemonReply, NodeDropEvent, NodeEvent},
node_to_daemon::DaemonRequest, node_to_daemon::DaemonRequest,
DataflowId,
}; };
use eyre::{eyre, Context};
use futures::{future, task, Future};
use eyre::{Context, eyre};
use futures::{Future, future, task};
use shared_memory_server::{ShmemConf, ShmemServer}; use shared_memory_server::{ShmemConf, ShmemServer};
use std::{ use std::{
collections::{BTreeMap, VecDeque}, collections::{BTreeMap, VecDeque},
@@ -50,7 +50,7 @@ pub async fn spawn_listener_loop(
Err(err) => { Err(err) => {
return Err( return Err(
eyre::Report::new(err).wrap_err("failed to create local TCP listener") eyre::Report::new(err).wrap_err("failed to create local TCP listener")
)
);
} }
}; };
let socket_addr = socket let socket_addr = socket
@@ -157,7 +157,7 @@ pub async fn spawn_listener_loop(
Ok(socket) => socket, Ok(socket) => socket,
Err(err) => { Err(err) => {
return Err(eyre::Report::new(err) return Err(eyre::Report::new(err)
.wrap_err("failed to create local Unix domain socket"))
.wrap_err("failed to create local Unix domain socket"));
} }
}; };




+ 2
- 2
binaries/daemon/src/node_communication/tcp.rs View File

@@ -2,8 +2,8 @@ use std::{collections::BTreeMap, io::ErrorKind, sync::Arc};


use super::{Connection, Listener}; use super::{Connection, Listener};
use crate::{ use crate::{
socket_stream_utils::{socket_stream_receive, socket_stream_send},
Event, Event,
socket_stream_utils::{socket_stream_receive, socket_stream_send},
}; };
use dora_core::{config::DataId, uhlc::HLC}; use dora_core::{config::DataId, uhlc::HLC};
use dora_message::{ use dora_message::{
@@ -70,7 +70,7 @@ impl Connection for TcpConnection {
| ErrorKind::ConnectionReset => return Ok(None), | ErrorKind::ConnectionReset => return Ok(None),
_other => { _other => {
return Err(err) return Err(err)
.context("unexpected I/O error while trying to receive DaemonRequest")
.context("unexpected I/O error while trying to receive DaemonRequest");
} }
}, },
}; };


+ 2
- 2
binaries/daemon/src/node_communication/unix_domain.rs View File

@@ -11,8 +11,8 @@ use tokio::{
}; };


use crate::{ use crate::{
socket_stream_utils::{socket_stream_receive, socket_stream_send},
Event, Event,
socket_stream_utils::{socket_stream_receive, socket_stream_send},
}; };


use super::{Connection, Listener}; use super::{Connection, Listener};
@@ -68,7 +68,7 @@ impl Connection for UnixConnection {
| ErrorKind::ConnectionReset => return Ok(None), | ErrorKind::ConnectionReset => return Ok(None),
_other => { _other => {
return Err(err) return Err(err)
.context("unexpected I/O error while trying to receive DaemonRequest")
.context("unexpected I/O error while trying to receive DaemonRequest");
} }
}, },
}; };


+ 4
- 4
binaries/daemon/src/pending.rs View File

@@ -2,18 +2,18 @@ use std::collections::{BTreeSet, HashMap, HashSet};


use dora_core::{ use dora_core::{
config::NodeId, config::NodeId,
uhlc::{Timestamp, HLC},
uhlc::{HLC, Timestamp},
}; };
use dora_message::{ use dora_message::{
DataflowId,
common::DaemonId, common::DaemonId,
daemon_to_coordinator::{CoordinatorRequest, DaemonEvent, LogLevel, LogMessage, Timestamped}, daemon_to_coordinator::{CoordinatorRequest, DaemonEvent, LogLevel, LogMessage, Timestamped},
daemon_to_node::DaemonReply, daemon_to_node::DaemonReply,
DataflowId,
}; };
use eyre::{bail, Context};
use eyre::{Context, bail};
use tokio::{net::TcpStream, sync::oneshot}; use tokio::{net::TcpStream, sync::oneshot};


use crate::{log::DataflowLogger, socket_stream_utils::socket_stream_send, CascadingErrorCauses};
use crate::{CascadingErrorCauses, log::DataflowLogger, socket_stream_utils::socket_stream_send};


pub struct PendingNodes { pub struct PendingNodes {
dataflow_id: DataflowId, dataflow_id: DataflowId,


+ 9
- 4
binaries/runtime/src/lib.rs View File

@@ -8,10 +8,10 @@ use dora_message::daemon_to_node::{NodeConfig, RuntimeConfig};
use dora_metrics::run_metrics_monitor; use dora_metrics::run_metrics_monitor;
use dora_node_api::{DoraNode, Event}; use dora_node_api::{DoraNode, Event};
use dora_tracing::TracingBuilder; use dora_tracing::TracingBuilder;
use eyre::{bail, Context, Result};
use eyre::{Context, Result, bail};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
use futures_concurrency::stream::Merge; use futures_concurrency::stream::Merge;
use operator::{run_operator, OperatorEvent, StopReason};
use operator::{OperatorEvent, StopReason, run_operator};


use std::{ use std::{
collections::{BTreeMap, BTreeSet, HashMap}, collections::{BTreeMap, BTreeSet, HashMap},
@@ -211,7 +211,9 @@ async fn run(
OperatorEvent::AllocateOutputSample { len, sample: tx } => { OperatorEvent::AllocateOutputSample { len, sample: tx } => {
let sample = node.allocate_data_sample(len); let sample = node.allocate_data_sample(len);
if tx.send(sample).is_err() { if tx.send(sample).is_err() {
tracing::warn!("output sample requested, but operator {operator_id} exited already");
tracing::warn!(
"output sample requested, but operator {operator_id} exited already"
);
} }
} }
OperatorEvent::Output { OperatorEvent::Output {
@@ -309,7 +311,10 @@ async fn run(
open_inputs.remove(&input_id); open_inputs.remove(&input_id);
if open_inputs.is_empty() { if open_inputs.is_empty() {
// all inputs of the node were closed -> close its event channel // all inputs of the node were closed -> close its event channel
tracing::trace!("all inputs of operator {}/{operator_id} were closed -> closing event channel", node.id());
tracing::trace!(
"all inputs of operator {}/{operator_id} were closed -> closing event channel",
node.id()
);
open_operator_inputs.remove(&operator_id); open_operator_inputs.remove(&operator_id);
operator_channels.remove(&operator_id); operator_channels.remove(&operator_id);
} }


+ 1
- 1
binaries/runtime/src/operator/channel.rs View File

@@ -1,8 +1,8 @@
use dora_core::config::DataId; use dora_core::config::DataId;
use dora_node_api::Event; use dora_node_api::Event;
use futures::{ use futures::{
future::{self, FusedFuture},
FutureExt, FutureExt,
future::{self, FusedFuture},
}; };
use std::collections::{BTreeMap, VecDeque}; use std::collections::{BTreeMap, VecDeque};




+ 8
- 10
binaries/runtime/src/operator/python.rs View File

@@ -3,21 +3,20 @@
use super::{OperatorEvent, StopReason}; use super::{OperatorEvent, StopReason};
use dora_core::{ use dora_core::{
config::{NodeId, OperatorId}, config::{NodeId, OperatorId},
descriptor::{source_is_url, Descriptor, PythonSource},
descriptor::{Descriptor, PythonSource, source_is_url},
}; };
use dora_download::download_file; use dora_download::download_file;
use dora_node_api::{merged::MergedEvent, Event, Parameter};
use dora_node_api::{Event, Parameter, merged::MergedEvent};
use dora_operator_api_python::PyEvent; use dora_operator_api_python::PyEvent;
use dora_operator_api_types::DoraStatus; use dora_operator_api_types::DoraStatus;
use eyre::{bail, eyre, Context, Result};
use eyre::{Context, Result, bail, eyre};
use pyo3::ffi::c_str; use pyo3::ffi::c_str;
use pyo3::{ use pyo3::{
pyclass,
Py, PyAny, Python, pyclass,
types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods, PyTracebackMethods}, types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods, PyTracebackMethods},
Py, PyAny, Python,
}; };
use std::{ use std::{
panic::{catch_unwind, AssertUnwindSafe},
panic::{AssertUnwindSafe, catch_unwind},
path::Path, path::Path,
}; };
use tokio::sync::{mpsc::Sender, oneshot}; use tokio::sync::{mpsc::Sender, oneshot};
@@ -295,16 +294,15 @@ mod callback_impl {
use dora_core::metadata::ArrowTypeInfoExt; use dora_core::metadata::ArrowTypeInfoExt;
use dora_message::metadata::ArrowTypeInfo; use dora_message::metadata::ArrowTypeInfo;
use dora_node_api::{ use dora_node_api::{
arrow_utils::{copy_array_into_sample, required_data_size},
ZERO_COPY_THRESHOLD, ZERO_COPY_THRESHOLD,
arrow_utils::{copy_array_into_sample, required_data_size},
}; };
use dora_operator_api_python::pydict_to_metadata; use dora_operator_api_python::pydict_to_metadata;
use dora_tracing::telemetry::deserialize_context; use dora_tracing::telemetry::deserialize_context;
use eyre::{eyre, Context, Result};
use eyre::{Context, Result, eyre};
use pyo3::{ use pyo3::{
pymethods,
Bound, PyObject, Python, pymethods,
types::{PyBytes, PyBytesMethods, PyDict}, types::{PyBytes, PyBytesMethods, PyDict},
Bound, PyObject, Python,
}; };
use tokio::sync::oneshot; use tokio::sync::oneshot;
use tracing::{field, span}; use tracing::{field, span};


+ 5
- 5
binaries/runtime/src/operator/shared_lib.rs View File

@@ -7,19 +7,19 @@ use dora_core::{
}; };
use dora_download::download_file; use dora_download::download_file;
use dora_node_api::{ use dora_node_api::{
arrow_utils::{copy_array_into_sample, required_data_size},
Event, Parameter, Event, Parameter,
arrow_utils::{copy_array_into_sample, required_data_size},
}; };
use dora_operator_api_types::{ use dora_operator_api_types::{
safer_ffi::closure::ArcDynFn1, DoraDropOperator, DoraInitOperator, DoraInitResult, DoraOnEvent,
DoraResult, DoraStatus, Metadata, OnEventResult, Output, SendOutput,
DoraDropOperator, DoraInitOperator, DoraInitResult, DoraOnEvent, DoraResult, DoraStatus,
Metadata, OnEventResult, Output, SendOutput, safer_ffi::closure::ArcDynFn1,
}; };
use eyre::{bail, eyre, Context, Result};
use eyre::{Context, Result, bail, eyre};
use libloading::Symbol; use libloading::Symbol;
use std::{ use std::{
collections::BTreeMap, collections::BTreeMap,
ffi::c_void, ffi::c_void,
panic::{catch_unwind, AssertUnwindSafe},
panic::{AssertUnwindSafe, catch_unwind},
path::Path, path::Path,
sync::Arc, sync::Arc,
}; };


+ 1
- 1
examples/benchmark/node/src/main.rs View File

@@ -1,4 +1,4 @@
use dora_node_api::{self, dora_core::config::DataId, DoraNode};
use dora_node_api::{self, DoraNode, dora_core::config::DataId};
use eyre::Context; use eyre::Context;
use rand::RngCore; use rand::RngCore;
use std::time::Duration; use std::time::Duration;


+ 1
- 1
examples/benchmark/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 4
- 2
examples/c++-arrow-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{env::consts::EXE_SUFFIX, path::Path, process::Command}; use std::{env::consts::EXE_SUFFIX, path::Path, process::Command};


struct ArrowConfig { struct ArrowConfig {
@@ -77,7 +77,9 @@ fn find_arrow_config() -> eyre::Result<ArrowConfig> {
.wrap_err("Failed to run pkg-config. Make sure Arrow C++ is installed")?; .wrap_err("Failed to run pkg-config. Make sure Arrow C++ is installed")?;


if !output.status.success() { if !output.status.success() {
bail!("Arrow C++ not found via pkg-config. Make sure it's installed and in your PKG_CONFIG_PATH");
bail!(
"Arrow C++ not found via pkg-config. Make sure it's installed and in your PKG_CONFIG_PATH"
);
} }


let cflags = String::from_utf8(output.stdout)?.trim().to_string(); let cflags = String::from_utf8(output.stdout)?.trim().to_string();


+ 1
- 1
examples/c++-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX}, env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX},
path::Path, path::Path,


+ 1
- 1
examples/c++-ros2-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{env::consts::EXE_SUFFIX, path::Path}; use std::{env::consts::EXE_SUFFIX, path::Path};


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/c-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::{ use std::{
env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX}, env::consts::{DLL_PREFIX, DLL_SUFFIX, EXE_SUFFIX},
path::Path, path::Path,


+ 1
- 1
examples/camera/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/cmake-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/multiple-daemons/node/src/main.rs View File

@@ -1,4 +1,4 @@
use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event, IntoArrow};
use dora_node_api::{self, DoraNode, Event, IntoArrow, dora_core::config::DataId};


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
println!("hello"); println!("hello");


+ 1
- 1
examples/multiple-daemons/operator/src/lib.rs View File

@@ -1,7 +1,7 @@
#![warn(unsafe_op_in_unsafe_fn)] #![warn(unsafe_op_in_unsafe_fn)]


use dora_operator_api::{ use dora_operator_api::{
register_operator, DoraOperator, DoraOutputSender, DoraStatus, Event, IntoArrow,
DoraOperator, DoraOutputSender, DoraStatus, Event, IntoArrow, register_operator,
}; };


register_operator!(ExampleOperator); register_operator!(ExampleOperator);


+ 2
- 2
examples/multiple-daemons/run.rs View File

@@ -1,7 +1,7 @@
use dora_cli::session::DataflowSession; use dora_cli::session::DataflowSession;
use dora_coordinator::{ControlEvent, Event}; use dora_coordinator::{ControlEvent, Event};
use dora_core::{ use dora_core::{
descriptor::{read_as_descriptor, DescriptorExt},
descriptor::{DescriptorExt, read_as_descriptor},
topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, DORA_COORDINATOR_PORT_DEFAULT}, topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, DORA_COORDINATOR_PORT_DEFAULT},
}; };
use dora_message::{ use dora_message::{
@@ -10,7 +10,7 @@ use dora_message::{
coordinator_to_cli::{ControlRequestReply, DataflowIdAndName}, coordinator_to_cli::{ControlRequestReply, DataflowIdAndName},
}; };
use dora_tracing::TracingBuilder; use dora_tracing::TracingBuilder;
use eyre::{bail, Context};
use eyre::{Context, bail};


use std::{ use std::{
collections::BTreeSet, collections::BTreeSet,


+ 4
- 2
examples/multiple-daemons/sink/src/main.rs View File

@@ -1,5 +1,5 @@
use dora_node_api::{self, DoraNode, Event}; use dora_node_api::{self, DoraNode, Event};
use eyre::{bail, Context};
use eyre::{Context, bail};


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
let (_node, mut events) = DoraNode::init_from_env()?; let (_node, mut events) = DoraNode::init_from_env()?;
@@ -16,7 +16,9 @@ fn main() -> eyre::Result<()> {
TryFrom::try_from(&data).context("expected string message")?; TryFrom::try_from(&data).context("expected string message")?;
println!("sink received message: {received_string}"); println!("sink received message: {received_string}");
if !received_string.starts_with("operator received random value ") { if !received_string.starts_with("operator received random value ") {
bail!("unexpected message format (should start with 'operator received random value')")
bail!(
"unexpected message format (should start with 'operator received random value')"
)
} }
if !received_string.ends_with(" ticks") { if !received_string.ends_with(" ticks") {
bail!("unexpected message format (should end with 'ticks')") bail!("unexpected message format (should end with 'ticks')")


+ 1
- 1
examples/python-dataflow/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/python-operator-dataflow/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/python-ros2-dataflow/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/rerun-viewer/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/rust-dataflow-git/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/rust-dataflow-url/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/rust-dataflow/node/src/main.rs View File

@@ -1,4 +1,4 @@
use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event, IntoArrow};
use dora_node_api::{self, DoraNode, Event, IntoArrow, dora_core::config::DataId};


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
println!("hello"); println!("hello");


+ 1
- 1
examples/rust-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 5
- 3
examples/rust-dataflow/sink-dynamic/src/main.rs View File

@@ -1,5 +1,5 @@
use dora_node_api::{self, dora_core::config::NodeId, DoraNode, Event};
use eyre::{bail, Context};
use dora_node_api::{self, DoraNode, Event, dora_core::config::NodeId};
use eyre::{Context, bail};


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
let (_node, mut events) = let (_node, mut events) =
@@ -17,7 +17,9 @@ fn main() -> eyre::Result<()> {
TryFrom::try_from(&data).context("expected string message")?; TryFrom::try_from(&data).context("expected string message")?;
println!("sink received message: {received_string}"); println!("sink received message: {received_string}");
if !received_string.starts_with("operator received random value ") { if !received_string.starts_with("operator received random value ") {
bail!("unexpected message format (should start with 'operator received random value')")
bail!(
"unexpected message format (should start with 'operator received random value')"
)
} }
if !received_string.ends_with(" ticks") { if !received_string.ends_with(" ticks") {
bail!("unexpected message format (should end with 'ticks')") bail!("unexpected message format (should end with 'ticks')")


+ 4
- 2
examples/rust-dataflow/sink/src/main.rs View File

@@ -1,5 +1,5 @@
use dora_node_api::{self, DoraNode, Event}; use dora_node_api::{self, DoraNode, Event};
use eyre::{bail, Context};
use eyre::{Context, bail};


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
let (_node, mut events) = DoraNode::init_from_env()?; let (_node, mut events) = DoraNode::init_from_env()?;
@@ -16,7 +16,9 @@ fn main() -> eyre::Result<()> {
TryFrom::try_from(&data).context("expected string message")?; TryFrom::try_from(&data).context("expected string message")?;
println!("sink received message: {received_string}"); println!("sink received message: {received_string}");
if !received_string.starts_with("operator received random value ") { if !received_string.starts_with("operator received random value ") {
bail!("unexpected message format (should start with 'operator received random value')")
bail!(
"unexpected message format (should start with 'operator received random value')"
)
} }
if !received_string.ends_with(" ticks") { if !received_string.ends_with(" ticks") {
bail!("unexpected message format (should end with 'ticks')") bail!("unexpected message format (should end with 'ticks')")


+ 1
- 1
examples/rust-dataflow/status-node/src/main.rs View File

@@ -1,4 +1,4 @@
use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event, IntoArrow};
use dora_node_api::{self, DoraNode, Event, IntoArrow, dora_core::config::DataId};
use eyre::Context; use eyre::Context;


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {


+ 3
- 4
examples/rust-ros2-dataflow/node/src/main.rs View File

@@ -1,10 +1,9 @@
use std::time::Duration; use std::time::Duration;


use dora_node_api::{ use dora_node_api::{
self,
self, DoraNode, Event,
dora_core::config::DataId, dora_core::config::DataId,
merged::{MergeExternal, MergedEvent}, merged::{MergeExternal, MergedEvent},
DoraNode, Event,
}; };
use dora_ros2_bridge::{ use dora_ros2_bridge::{
messages::{ messages::{
@@ -12,10 +11,10 @@ use dora_ros2_bridge::{
geometry_msgs::msg::{Twist, Vector3}, geometry_msgs::msg::{Twist, Vector3},
turtlesim::msg::Pose, turtlesim::msg::Pose,
}, },
ros2_client::{self, ros2, NodeOptions},
ros2_client::{self, NodeOptions, ros2},
rustdds::{self, policy}, rustdds::{self, policy},
}; };
use eyre::{eyre, Context};
use eyre::{Context, eyre};
use futures::task::SpawnExt; use futures::task::SpawnExt;


fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {


+ 1
- 1
examples/rust-ros2-dataflow/run.rs View File

@@ -1,5 +1,5 @@
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, Context};
use eyre::{Context, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
examples/vlm/run.rs View File

@@ -1,6 +1,6 @@
use dora_core::{get_uv_path, run}; use dora_core::{get_uv_path, run};
use dora_tracing::set_up_tracing; use dora_tracing::set_up_tracing;
use eyre::{bail, WrapErr};
use eyre::{WrapErr, bail};
use std::path::Path; use std::path::Path;


#[tokio::main] #[tokio::main]


+ 1
- 1
libraries/arrow-convert/src/from_impls.rs View File

@@ -274,7 +274,7 @@ where


#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use arrow::array::{make_array, PrimitiveArray};
use arrow::array::{PrimitiveArray, make_array};


use crate::ArrowData; use crate::ArrowData;




+ 2
- 2
libraries/arrow-convert/src/into_impls.rs View File

@@ -1,8 +1,8 @@
use crate::IntoArrow; use crate::IntoArrow;
use arrow::array::{PrimitiveArray, StringArray, TimestampNanosecondArray}; use arrow::array::{PrimitiveArray, StringArray, TimestampNanosecondArray};
use arrow::datatypes::{ use arrow::datatypes::{
ArrowPrimitiveType, ArrowTimestampType, Float16Type, Float32Type, Float64Type, Int16Type,
Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
ArrowPrimitiveType, ArrowTimestampType, Float16Type, Float32Type, Float64Type, Int8Type,
Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
}; };
use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
use half::f16; use half::f16;


+ 3
- 3
libraries/arrow-convert/src/lib.rs View File

@@ -3,11 +3,11 @@
#![warn(missing_docs)] #![warn(missing_docs)]


use arrow::array::{ use arrow::array::{
Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, UInt16Array,
UInt32Array, UInt8Array,
Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, UInt8Array,
UInt16Array, UInt32Array,
}; };
use arrow::datatypes::DataType; use arrow::datatypes::DataType;
use eyre::{eyre, ContextCompat, Result};
use eyre::{ContextCompat, Result, eyre};
use num::NumCast; use num::NumCast;
use std::ops::{Deref, DerefMut}; use std::ops::{Deref, DerefMut};




+ 1
- 1
libraries/communication-layer/pub-sub/src/zenoh.rs View File

@@ -4,7 +4,7 @@ use super::{CommunicationLayer, Publisher, Subscriber};
use crate::{BoxError, ReceivedSample}; use crate::{BoxError, ReceivedSample};
use std::{borrow::Cow, sync::Arc, time::Duration}; use std::{borrow::Cow, sync::Arc, time::Duration};
use zenoh::{ use zenoh::{
prelude::{sync::SyncResolve, Config, Priority, SessionDeclarations, SplitBuffer},
prelude::{Config, Priority, SessionDeclarations, SplitBuffer, sync::SyncResolve},
publication::CongestionControl, publication::CongestionControl,
}; };




+ 8
- 8
libraries/communication-layer/request-reply/src/lib.rs View File

@@ -28,10 +28,10 @@ pub trait RequestReplyLayer: Send + Sync {
Item = Result< Item = Result<
Box< Box<
dyn ListenConnection< dyn ListenConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
>, >,
Self::Error, Self::Error,
>, >,
@@ -47,10 +47,10 @@ pub trait RequestReplyLayer: Send + Sync {
) -> Result< ) -> Result<
Box< Box<
dyn RequestReplyConnection< dyn RequestReplyConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
>, >,
Self::Error, Self::Error,
>; >;


+ 16
- 16
libraries/communication-layer/request-reply/src/tcp.rs View File

@@ -37,10 +37,10 @@ impl RequestReplyLayer for TcpLayer {
Item = Result< Item = Result<
Box< Box<
dyn crate::ListenConnection< dyn crate::ListenConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
>, >,
Self::Error, Self::Error,
>, >,
@@ -56,10 +56,10 @@ impl RequestReplyLayer for TcpLayer {
r.map(|stream| { r.map(|stream| {
let connection: Box< let connection: Box<
dyn ListenConnection< dyn ListenConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
> = Box::new(TcpConnection { stream }); > = Box::new(TcpConnection { stream });
connection connection
}) })
@@ -74,20 +74,20 @@ impl RequestReplyLayer for TcpLayer {
) -> Result< ) -> Result<
Box< Box<
dyn crate::RequestReplyConnection< dyn crate::RequestReplyConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
>, >,
Self::Error, Self::Error,
> { > {
TcpStream::connect(addr).map(|s| { TcpStream::connect(addr).map(|s| {
let connection: Box< let connection: Box<
dyn RequestReplyConnection< dyn RequestReplyConnection<
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
RequestData = Self::RequestData,
ReplyData = Self::ReplyData,
Error = Self::Error,
>,
> = Box::new(TcpConnection { stream: s }); > = Box::new(TcpConnection { stream: s });
connection connection
}) })


+ 1
- 1
libraries/core/src/build/build_command.rs View File

@@ -6,7 +6,7 @@ use std::{
}; };


use dora_message::descriptor::EnvValue; use dora_message::descriptor::EnvValue;
use eyre::{eyre, Context};
use eyre::{Context, eyre};


pub fn run_build_command( pub fn run_build_command(
build: &str, build: &str,


+ 2
- 2
libraries/core/src/build/git.rs View File

@@ -1,6 +1,6 @@
use crate::build::{BuildLogger, PrevGitSource}; use crate::build::{BuildLogger, PrevGitSource};
use dora_message::{common::LogLevel, DataflowId, SessionId};
use eyre::{bail, ContextCompat, WrapErr};
use dora_message::{DataflowId, SessionId, common::LogLevel};
use eyre::{ContextCompat, WrapErr, bail};
use git2::FetchOptions; use git2::FetchOptions;
use itertools::Itertools; use itertools::Itertools;
use std::{ use std::{


+ 7
- 5
libraries/core/src/descriptor/mod.rs View File

@@ -3,7 +3,7 @@ use dora_message::{
descriptor::{GitRepoRev, NodeSource}, descriptor::{GitRepoRev, NodeSource},
id::{DataId, NodeId, OperatorId}, id::{DataId, NodeId, OperatorId},
}; };
use eyre::{bail, Context, OptionExt, Result};
use eyre::{Context, OptionExt, Result, bail};
use std::{ use std::{
collections::{BTreeMap, HashMap}, collections::{BTreeMap, HashMap},
env::consts::EXE_EXTENSION, env::consts::EXE_EXTENSION,
@@ -14,9 +14,9 @@ use tokio::process::Command;


// reexport for compatibility // reexport for compatibility
pub use dora_message::descriptor::{ pub use dora_message::descriptor::{
CoreNodeKind, CustomNode, Descriptor, Node, OperatorConfig, OperatorDefinition, OperatorSource,
PythonSource, ResolvedNode, RuntimeNode, SingleOperatorDefinition, DYNAMIC_SOURCE,
SHELL_SOURCE,
CoreNodeKind, CustomNode, DYNAMIC_SOURCE, Descriptor, Node, OperatorConfig, OperatorDefinition,
OperatorSource, PythonSource, ResolvedNode, RuntimeNode, SHELL_SOURCE,
SingleOperatorDefinition,
}; };
pub use validate::ResolvedNodeExt; pub use validate::ResolvedNodeExt;
pub use visualize::collect_dora_timers; pub use visualize::collect_dora_timers;
@@ -165,7 +165,9 @@ fn node_kind_mut(node: &mut Node) -> eyre::Result<NodeKindMut> {
(None, Some(tag), None) => Some(GitRepoRev::Tag(tag.clone())), (None, Some(tag), None) => Some(GitRepoRev::Tag(tag.clone())),
(None, None, Some(rev)) => Some(GitRepoRev::Rev(rev.clone())), (None, None, Some(rev)) => Some(GitRepoRev::Rev(rev.clone())),
other @ (_, _, _) => { other @ (_, _, _) => {
eyre::bail!("only one of `branch`, `tag`, and `rev` are allowed (got {other:?})")
eyre::bail!(
"only one of `branch`, `tag`, and `rev` are allowed (got {other:?})"
)
} }
}; };
NodeSource::GitBranch { NodeSource::GitBranch {


+ 1
- 1
libraries/core/src/descriptor/visualize.rs View File

@@ -1,5 +1,5 @@
use dora_message::{ use dora_message::{
config::{format_duration, Input, InputMapping, UserInputMapping},
config::{Input, InputMapping, UserInputMapping, format_duration},
descriptor::{CoreNodeKind, OperatorDefinition}, descriptor::{CoreNodeKind, OperatorDefinition},
id::{DataId, NodeId}, id::{DataId, NodeId},
}; };


+ 1
- 1
libraries/core/src/lib.rs View File

@@ -1,4 +1,4 @@
use eyre::{bail, eyre, Context};
use eyre::{Context, bail, eyre};
use std::{ use std::{
env::consts::{DLL_PREFIX, DLL_SUFFIX}, env::consts::{DLL_PREFIX, DLL_SUFFIX},
ffi::OsStr, ffi::OsStr,


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/parser/action.rs View File

@@ -61,7 +61,7 @@ mod test {
use std::path::PathBuf; use std::path::PathBuf;


use super::*; use super::*;
use crate::types::{primitives::*, sequences::*, MemberType};
use crate::types::{MemberType, primitives::*, sequences::*};


fn parse_action_def(srv_name: &str) -> Result<Action> { fn parse_action_def(srv_name: &str) -> Result<Action> {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))


+ 2
- 2
libraries/extensions/ros2-bridge/msg-gen/src/parser/constant.rs View File

@@ -1,4 +1,4 @@
use anyhow::{ensure, Result};
use anyhow::{Result, ensure};
use nom::{ use nom::{
bytes::complete::is_not, bytes::complete::is_not,
character::complete::{char, space0, space1}, character::complete::{char, space0, space1},
@@ -8,7 +8,7 @@ use nom::{
}; };


use super::{error::RclMsgError, ident, literal, types}; use super::{error::RclMsgError, ident, literal, types};
use crate::types::{primitives::PrimitiveType, Constant, ConstantType};
use crate::types::{Constant, ConstantType, primitives::PrimitiveType};


fn validate_value(r#type: ConstantType, value: &str) -> Result<Vec<String>> { fn validate_value(r#type: ConstantType, value: &str) -> Result<Vec<String>> {
match r#type { match r#type {


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/parser/ident.rs View File

@@ -1,10 +1,10 @@
use nom::{ use nom::{
IResult,
branch::alt, branch::alt,
character::complete::{alphanumeric0, char, one_of}, character::complete::{alphanumeric0, char, one_of},
combinator::{opt, recognize}, combinator::{opt, recognize},
multi::{many1, separated_list0, separated_list1}, multi::{many1, separated_list0, separated_list1},
sequence::{pair, tuple}, sequence::{pair, tuple},
IResult,
}; };


fn upperalpha(s: &str) -> IResult<&str, char> { fn upperalpha(s: &str) -> IResult<&str, char> {


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/parser/literal.rs View File

@@ -1,6 +1,7 @@
use std::convert::TryFrom; use std::convert::TryFrom;


use nom::{ use nom::{
IResult,
branch::alt, branch::alt,
bytes::complete::{is_not, tag, tag_no_case, take_while}, bytes::complete::{is_not, tag, tag_no_case, take_while},
character::complete::{anychar, char, digit1, hex_digit1, none_of, oct_digit1, one_of, space0}, character::complete::{anychar, char, digit1, hex_digit1, none_of, oct_digit1, one_of, space0},
@@ -8,7 +9,6 @@ use nom::{
multi::{many0, separated_list1}, multi::{many0, separated_list1},
number::complete::recognize_float, number::complete::recognize_float,
sequence::{delimited, pair, tuple}, sequence::{delimited, pair, tuple},
IResult,
}; };


use crate::types::primitives::{BasicType, GenericString}; use crate::types::primitives::{BasicType, GenericString};


+ 2
- 2
libraries/extensions/ros2-bridge/msg-gen/src/parser/member.rs View File

@@ -1,4 +1,4 @@
use anyhow::{ensure, Result};
use anyhow::{Result, ensure};
use nom::{ use nom::{
bytes::complete::is_not, bytes::complete::is_not,
character::complete::{space0, space1}, character::complete::{space0, space1},
@@ -8,7 +8,7 @@ use nom::{
}; };


use super::{error::RclMsgError, ident, literal, types}; use super::{error::RclMsgError, ident, literal, types};
use crate::types::{primitives::NestableType, Member, MemberType};
use crate::types::{Member, MemberType, primitives::NestableType};


fn nestable_type_default(nestable_type: NestableType, default: &str) -> Result<Vec<String>> { fn nestable_type_default(nestable_type: NestableType, default: &str) -> Result<Vec<String>> {
match nestable_type { match nestable_type {


+ 3
- 3
libraries/extensions/ros2-bridge/msg-gen/src/parser/package.rs View File

@@ -51,9 +51,9 @@ fn get_ros_msgs_each_package<P: AsRef<Path>>(root_dir: P) -> Result<Vec<Package>
continue; continue;
} else if visited_files.contains(&(package.clone(), file_name.clone())) { } else if visited_files.contains(&(package.clone(), file_name.clone())) {
warn!( warn!(
"found two versions of package: {:?}, message: {:?}. will skip the one in: {:#?}",
package, file_name, path
);
"found two versions of package: {:?}, message: {:?}. will skip the one in: {:#?}",
package, file_name, path
);
continue; continue;
} else { } else {
visited_files.push((package.clone(), file_name.clone())); visited_files.push((package.clone(), file_name.clone()));


+ 2
- 2
libraries/extensions/ros2-bridge/msg-gen/src/parser/types.rs View File

@@ -1,11 +1,11 @@
use anyhow::anyhow; use anyhow::anyhow;
use nom::{ use nom::{
IResult,
branch::alt, branch::alt,
bytes::complete::tag, bytes::complete::tag,
character::complete::{char, space1}, character::complete::{char, space1},
combinator::{eof, map, map_res, opt, peek}, combinator::{eof, map, map_res, opt, peek},
sequence::{delimited, pair, preceded, tuple}, sequence::{delimited, pair, preceded, tuple},
IResult,
}; };


use super::{ use super::{
@@ -13,9 +13,9 @@ use super::{
literal::usize_literal, literal::usize_literal,
}; };
use crate::types::{ use crate::types::{
ConstantType, MemberType,
primitives::*, primitives::*,
sequences::{Array, BoundedSequence, PrimitiveArray, Sequence}, sequences::{Array, BoundedSequence, PrimitiveArray, Sequence},
ConstantType, MemberType,
}; };


pub fn parse_member_type(s: &str) -> IResult<&str, MemberType> { pub fn parse_member_type(s: &str) -> IResult<&str, MemberType> {


+ 2
- 2
libraries/extensions/ros2-bridge/msg-gen/src/types/action.rs View File

@@ -1,8 +1,8 @@
use heck::SnakeCase; use heck::SnakeCase;
use quote::{format_ident, quote, ToTokens};
use quote::{ToTokens, format_ident, quote};
use syn::Ident; use syn::Ident;


use super::{primitives::*, Member, Message, Service};
use super::{Member, Message, Service, primitives::*};


/// An action definition /// An action definition
#[derive(Debug, Clone)] #[derive(Debug, Clone)]


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/types/constant.rs View File

@@ -1,4 +1,4 @@
use quote::{quote, ToTokens};
use quote::{ToTokens, quote};


use super::{ use super::{
primitives::{BasicType, GenericUnboundedString, PrimitiveType}, primitives::{BasicType, GenericUnboundedString, PrimitiveType},


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/types/member.rs View File

@@ -1,4 +1,4 @@
use quote::{quote, ToTokens};
use quote::{ToTokens, quote};


use super::{primitives::*, sequences::*}; use super::{primitives::*, sequences::*};




+ 2
- 2
libraries/extensions/ros2-bridge/msg-gen/src/types/message.rs View File

@@ -1,7 +1,7 @@
use quote::{format_ident, quote, ToTokens};
use quote::{ToTokens, format_ident, quote};
use syn::Ident; use syn::Ident;


use super::{primitives::*, sequences::Array, ConstantType, MemberType};
use super::{ConstantType, MemberType, primitives::*, sequences::Array};


/// A member of a structure /// A member of a structure
#[derive(Debug, Clone)] #[derive(Debug, Clone)]


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/types/package.rs View File

@@ -1,5 +1,5 @@
use proc_macro2::Span; use proc_macro2::Span;
use quote::{quote, ToTokens};
use quote::{ToTokens, quote};
use syn::Ident; use syn::Ident;


use crate::types::{Action, Message, Service}; use crate::types::{Action, Message, Service};


+ 1
- 1
libraries/extensions/ros2-bridge/msg-gen/src/types/primitives.rs View File

@@ -1,7 +1,7 @@
use std::fmt; use std::fmt;


use proc_macro2::{Ident, Literal, Span}; use proc_macro2::{Ident, Literal, Span};
use quote::{format_ident, quote, ToTokens};
use quote::{ToTokens, format_ident, quote};


macro_rules! define_enum_from { macro_rules! define_enum_from {
($into_t:ty, $from_t:ty, $path:path) => { ($into_t:ty, $from_t:ty, $path:path) => {


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save