Remove code duplicate for tracing subscriber and use env variable to manage log level.

The intent of this commit is to remove the quantity of log that is being pushed to user.

This commit removes the redeclaration of a set up tracing methods to centralise
the tokio-tracing subscriber within the extension crate. It also add the
feature to filter information based on Environment variable.
This makes it possible to define the log level for tokio tracing like
this:
```
RUST_LOG=debug dora-daemon --run-dataflow dataflow.yml
```

I have also unified the feature flag to make it easier to manage tracing features among the workspace.

I did not change the default behaviour of tracing in our crates and therefore by
using the command above you should get the same tracing log as before.

fix merge conflict generated

Cargo.lock

@@ -949,6 +949,7 @@ dependencies = [
  "dora-core",
  "dora-download",
  "dora-node-api",
+ "dora-tracing",
  "eyre",
  "futures",
  "futures-concurrency",
@@ -962,7 +963,6 @@ dependencies = [
  "tokio-stream",
  "tokio-util 0.7.1",
  "tracing",
- "tracing-subscriber",
  "uuid 1.2.1",
  "which",
  "zenoh",
@@ -994,6 +994,7 @@ dependencies = [
  "ctrlc",
  "dora-core",
  "dora-download",
+ "dora-tracing",
  "eyre",
  "flume",
  "futures",
@@ -1005,7 +1006,6 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tracing",
- "tracing-subscriber",
  "uuid 1.2.1",
 ]
 
@@ -1063,6 +1063,7 @@ dependencies = [
  "bincode",
  "capnp",
  "dora-core",
+ "dora-tracing",
  "eyre",
  "flume",
  "once_cell",
@@ -1074,7 +1075,6 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
- "tracing-subscriber",
  "uuid 1.2.1",
 ]
 
@@ -1198,9 +1198,12 @@ dependencies = [
 name = "dora-tracing"
 version = "0.1.3"
 dependencies = [
+ "eyre",
  "opentelemetry",
  "opentelemetry-jaeger",
  "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -1999,6 +2002,15 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
+[[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata",
+]
+
 [[package]]
 name = "matches"
 version = "0.1.9"
@@ -3130,6 +3142,15 @@ dependencies = [
  "regex-syntax",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax",
+]
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.25"
@@ -4077,10 +4098,14 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
 dependencies = [
+ "matchers",
  "nu-ansi-term",
+ "once_cell",
+ "regex",
  "sharded-slab",
  "smallvec",
  "thread_local",
+ "tracing",
  "tracing-core",
  "tracing-log",
 ]

Cargo.toml

@@ -29,6 +29,7 @@ version = "0.1.3"
 dora-node-api = { version = "0.1.3", path = "apis/rust/node", default-features = false }
 dora-operator-api = { version = "0.1.3", path = "apis/rust/operator", default-features = false }
 dora-core = { version = "0.1.3", path = "libraries/core" }
+dora-tracing = { version = "0.1.3", path = "libraries/extensions/telemetry/tracing" }
 
 [package]
 name = "dora-examples"

apis/c++/node/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2021"
 crate-type = ["staticlib"]
 
 [features]
-default = ["tracing-subscriber"]
-tracing-subscriber = ["dora-node-api/tracing-subscriber"]
+default = ["tracing"]
+tracing = ["dora-node-api/tracing"]
 
 [dependencies]
 cxx = "1.0.73"

apis/c/node/Cargo.toml

@@ -10,8 +10,8 @@ license = "Apache-2.0"
 crate-type = ["staticlib"]
 
 [features]
-default = ["tracing-subscriber"]
-tracing-subscriber = ["dora-node-api/tracing-subscriber"]
+default = ["tracing"]
+tracing = ["dora-node-api/tracing"]
 
 [dependencies]
 eyre = "0.6.8"

apis/python/node/Cargo.toml

@@ -6,6 +6,10 @@ license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["tracing"]
+tracing = ["dora-node-api/tracing"]
+
 [dependencies]
 dora-node-api = { path = "../../rust/node" }
 dora-operator-api-python = { path = "../operator" }

apis/rust/node/Cargo.toml

@@ -5,8 +5,8 @@ edition = "2021"
 license = "Apache-2.0"
 
 [features]
-default = ["tracing-subscriber"]
-tracing-subscriber = ["dep:tracing-subscriber"]
+default = ["tracing"]
+tracing = ["dep:dora-tracing"]
 
 [dependencies]
 dora-core = { path = "../../../libraries/core" }
@@ -18,12 +18,12 @@ serde_yaml = "0.8.23"
 serde_json = "1.0.89"
 thiserror = "1.0.30"
 tracing = "0.1.33"
-tracing-subscriber = { version = "0.3.15", optional = true }
 flume = "0.10.14"
 uuid = { version = "1.1.2", features = ["v4"] }
 capnp = "0.14.11"
 bincode = "1.3.3"
 shared_memory = "0.12.0"
+dora-tracing = { workspace = true, optional = true }
 
 [dev-dependencies]
 tokio = { version = "1.24.2", features = ["rt"] }

apis/rust/node/src/node.rs

@@ -11,6 +11,9 @@ use crate::{
     EventStream,
 };
 
+#[cfg(feature = "tracing")]
+use dora_tracing::set_up_tracing;
+
 const ZERO_COPY_THRESHOLD: usize = 4096;
 
 pub struct DoraNode {
@@ -22,7 +25,7 @@ pub struct DoraNode {
 
 impl DoraNode {
     pub fn init_from_env() -> eyre::Result<(Self, EventStream)> {
-        #[cfg(feature = "tracing-subscriber")]
+        #[cfg(feature = "tracing")]
         set_up_tracing().context("failed to set up tracing subscriber")?;
 
         let node_config = {
@@ -131,13 +134,3 @@ impl Drop for DoraNode {
         }
     }
 }
-
-#[cfg(feature = "tracing-subscriber")]
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
-
-    let stdout_log = tracing_subscriber::fmt::layer().pretty();
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
-        .context("failed to set tracing global subscriber")
-}

binaries/coordinator/Cargo.toml

@@ -6,6 +6,10 @@ license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["tracing"]
+tracing = ["dep:dora-tracing"]
+
 [dependencies]
 bincode = "1.3.3"
 dora-node-api = { path = "../../apis/rust/node" }
@@ -22,11 +26,11 @@ time = "0.3.9"
 rand = "0.8.5"
 dora-core = { workspace = true }
 tracing = "0.1.36"
-tracing-subscriber = "0.3.15"
 futures-concurrency = "7.1.0"
 zenoh = { git = "https://github.com/eclipse-zenoh/zenoh.git", rev = "79a136e4fd90b11ff5d775ced981af53c4f1071b" }
 serde_json = "1.0.86"
 dora-download = { path = "../../libraries/extensions/download" }
+dora-tracing = { workspace = true, optional = true }
 which = "4.3.0"
 communication-layer-request-reply = { path = "../../libraries/communication-layer/request-reply" }
 thiserror = "1.0.37"

binaries/coordinator/src/main.rs

@@ -1,18 +1,13 @@
+#[cfg(feature = "tracing")]
+use dora_tracing::set_up_tracing;
+#[cfg(feature = "tracing")]
 use eyre::Context;
 
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
+    #[cfg(feature = "tracing")]
     set_up_tracing().context("failed to set up tracing subscriber")?;
 
     let args = clap::Parser::parse();
     dora_coordinator::run(args).await
 }
-
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
-
-    let stdout_log = tracing_subscriber::fmt::layer().pretty();
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
-        .context("failed to set tracing global subscriber")
-}

binaries/daemon/Cargo.toml

@@ -5,18 +5,22 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["tracing"]
+tracing = ["dep:dora-tracing"]
+
 [dependencies]
 eyre = "0.6.8"
 tokio = { version = "1.20.1", features = ["full"] }
 tokio-stream = { version = "0.1.11", features = ["net"] }
 tracing = "0.1.36"
-tracing-subscriber = "0.3.15"
 futures-concurrency = "7.1.0"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.86"
 dora-core = { path = "../../libraries/core" }
 flume = "0.10.14"
 dora-download = { path = "../../libraries/extensions/download" }
+dora-tracing = { workspace = true, optional = true }
 serde_yaml = "0.8.23"
 uuid = { version = "1.1.2", features = ["v4"] }
 futures = "0.3.25"

binaries/daemon/src/main.rs

@@ -1,9 +1,11 @@
 use dora_core::topics::DORA_COORDINATOR_PORT_DEFAULT;
 use dora_daemon::Daemon;
+#[cfg(feature = "tracing")]
+use dora_tracing::set_up_tracing;
+#[cfg(feature = "tracing")]
 use eyre::Context;
+
 use std::{net::Ipv4Addr, path::PathBuf};
-use tracing::metadata::LevelFilter;
-use tracing_subscriber::Layer;
 
 #[derive(Debug, Clone, clap::Parser)]
 #[clap(about = "Dora daemon")]
@@ -23,6 +25,7 @@ async fn main() -> eyre::Result<()> {
 }
 
 async fn run() -> eyre::Result<()> {
+    #[cfg(feature = "tracing")]
     set_up_tracing().wrap_err("failed to set up tracing subscriber")?;
 
     let Args {
@@ -47,14 +50,3 @@ async fn run() -> eyre::Result<()> {
         }
     }
 }
-
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
-
-    let stdout_log = tracing_subscriber::fmt::layer()
-        .pretty()
-        .with_filter(LevelFilter::DEBUG);
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
-        .context("failed to set tracing global subscriber")
-}

binaries/runtime/src/lib.rs

@@ -11,16 +11,18 @@ use futures::{Stream, StreamExt};
 use futures_concurrency::stream::Merge;
 use operator::{run_operator, OperatorEvent, StopReason};
 
+#[cfg(feature = "tracing")]
+use dora_tracing::set_up_tracing;
 use std::{
     collections::{BTreeSet, HashMap},
     mem,
 };
 use tokio::{runtime::Builder, sync::mpsc};
 use tokio_stream::wrappers::ReceiverStream;
-
 mod operator;
 
 pub fn main() -> eyre::Result<()> {
+    #[cfg(feature = "tracing")]
     set_up_tracing().context("failed to set up tracing subscriber")?;
 
     let config: RuntimeConfig = {
@@ -293,12 +295,3 @@ enum Event {
     InputClosed(dora_core::config::DataId),
     Error(String),
 }
-
-fn set_up_tracing() -> eyre::Result<()> {
-    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
-
-    let stdout_log = tracing_subscriber::fmt::layer().pretty();
-    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
-    tracing::subscriber::set_global_default(subscriber)
-        .context("failed to set tracing global subscriber")
-}

binaries/runtime/src/operator/shared_lib.rs

@@ -151,8 +151,8 @@ impl<'lib> SharedLibraryOperator<'lib> {
                     };
 
                     let span = tracer.start_with_context(
-                        format!("{}", input.id),
-                        &deserialize_context(&input.metadata.parameters.open_telemetry_context),
+                        format!("{}", input_id),
+                        &deserialize_context(&metadata.parameters.open_telemetry_context),
                     );
                     let child_cx = OtelContext::current_with_span(span);
                     let string_cx = serialize_context(&child_cx);

libraries/extensions/telemetry/tracing/Cargo.toml

@@ -10,3 +10,6 @@ license = "Apache-2.0"
 opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] }
 opentelemetry-jaeger = { version = "0.16", features = ["rt-tokio"] }
 tokio = { version = "1.24.2", features = ["full"] }
+tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
+eyre = "0.6.8"
+tracing = "0.1.36"

libraries/extensions/telemetry/tracing/src/lib.rs

@@ -5,10 +5,12 @@
 
 use std::collections::HashMap;
 
+use eyre::Context as EyreContext;
 use opentelemetry::propagation::Extractor;
 use opentelemetry::sdk::{propagation::TraceContextPropagator, trace as sdktrace};
 use opentelemetry::trace::TraceError;
 use opentelemetry::{global, Context};
+use tracing_subscriber::{EnvFilter, Layer};
 
 struct MetadataMap<'a>(HashMap<&'a str, &'a str>);
 
@@ -69,3 +71,15 @@ pub fn deserialize_context(string_context: &str) -> Context {
     }
     global::get_text_map_propagator(|prop| prop.extract(&map))
 }
+
+pub fn set_up_tracing() -> eyre::Result<()> {
+    use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
+    let filter = EnvFilter::from_default_env();
+
+    let stdout_log = tracing_subscriber::fmt::layer()
+        .pretty()
+        .with_filter(filter);
+    let subscriber = tracing_subscriber::Registry::default().with(stdout_log);
+    tracing::subscriber::set_global_default(subscriber)
+        .context("failed to set tracing global subscriber")
+}