dora-rs
/
dora
Not watched
1
0
Fork 0
Code
Releases 124 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Bump opentelemetry metrics to 0.21

tags/v0.3.1-rc
haixuanTao 2 years ago
parent
408d13a0af
commit
32831cae75
5 changed files with 185 additions and 195 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +162
    -168
      Cargo.lock
  2. +3
    -6
      binaries/runtime/Cargo.toml
  3. +6
    -3
      binaries/runtime/src/lib.rs
  4. +3
    -3
      libraries/extensions/telemetry/metrics/Cargo.toml
  5. +11
    -15
      libraries/extensions/telemetry/metrics/src/lib.rs

+ 162
- 168
Cargo.lock View File

@@ -497,28 +497,6 @@ dependencies = [
"wasm-bindgen-futures",
]

[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]

[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
]

[[package]]
name = "async-task"
version = "4.4.1"
@@ -559,6 +537,51 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

[[package]]
name = "axum"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
dependencies = [
"async-trait",
"axum-core",
"bitflags 1.3.2",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"tower",
"tower-layer",
"tower-service",
]

[[package]]
name = "axum-core"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]

[[package]]
name = "backtrace"
version = "0.3.69"
@@ -1248,16 +1271,6 @@ dependencies = [
"syn 2.0.38",
]

[[package]]
name = "dashmap"
version = "4.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
dependencies = [
"cfg-if 1.0.0",
"num_cpus",
]

[[package]]
name = "dashmap"
version = "5.5.3"
@@ -1490,9 +1503,9 @@ name = "dora-metrics"
version = "0.3.0"
dependencies = [
"futures",
"opentelemetry 0.17.0",
"opentelemetry 0.21.0",
"opentelemetry-otlp",
"opentelemetry-system-metrics",
"opentelemetry_sdk 0.21.1",
"tokio",
]

@@ -1712,7 +1725,7 @@ dependencies = [
"futures",
"futures-concurrency",
"libloading",
"opentelemetry 0.18.0",
"opentelemetry 0.21.0",
"opentelemetry-system-metrics",
"pyo3",
"pythonize",
@@ -2306,7 +2319,7 @@ dependencies = [
"indexmap 1.9.3",
"slab",
"tokio",
"tokio-util 0.7.9",
"tokio-util",
"tracing",
]

@@ -3028,6 +3041,12 @@ dependencies = [
"regex-automata 0.1.10",
]

[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"

[[package]]
name = "md5"
version = "0.7.0"
@@ -3152,12 +3171,6 @@ dependencies = [
"ws2_32-sys",
]

[[package]]
name = "multimap"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"

[[package]]
name = "multiple-daemons-example-node"
version = "0.3.0"
@@ -3522,35 +3535,28 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

[[package]]
name = "opentelemetry"
version = "0.17.0"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e"
dependencies = [
"async-trait",
"crossbeam-channel",
"dashmap 4.0.2",
"fnv",
"futures-channel",
"futures-executor",
"futures-util",
"js-sys",
"lazy_static",
"percent-encoding",
"pin-project",
"rand",
"thiserror",
"tokio",
"tokio-stream",
"opentelemetry_api",
"opentelemetry_sdk 0.18.0",
]

[[package]]
name = "opentelemetry"
version = "0.18.0"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e"
checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
dependencies = [
"opentelemetry_api",
"opentelemetry_sdk",
"futures-core",
"futures-sink",
"indexmap 2.0.2",
"js-sys",
"once_cell",
"pin-project-lite",
"thiserror",
"urlencoding",
]

[[package]]
@@ -3564,7 +3570,7 @@ dependencies = [
"futures-executor",
"once_cell",
"opentelemetry 0.18.0",
"opentelemetry-semantic-conventions",
"opentelemetry-semantic-conventions 0.10.0",
"thiserror",
"thrift",
"tokio",
@@ -3572,20 +3578,33 @@ dependencies = [

[[package]]
name = "opentelemetry-otlp"
version = "0.10.0"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"
checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930"
dependencies = [
"async-trait",
"futures",
"futures-util",
"futures-core",
"http",
"opentelemetry 0.17.0",
"opentelemetry 0.21.0",
"opentelemetry-proto",
"opentelemetry-semantic-conventions 0.13.0",
"opentelemetry_sdk 0.21.1",
"prost",
"thiserror",
"tokio",
"tonic",
"tonic-build",
]

[[package]]
name = "opentelemetry-proto"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1"
dependencies = [
"opentelemetry 0.21.0",
"opentelemetry_sdk 0.21.1",
"prost",
"tonic",
]

[[package]]
@@ -3597,14 +3616,23 @@ dependencies = [
"opentelemetry 0.18.0",
]

[[package]]
name = "opentelemetry-semantic-conventions"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84"
dependencies = [
"opentelemetry 0.21.0",
]

[[package]]
name = "opentelemetry-system-metrics"
version = "0.1.2"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d425a3381784debcefe6c609c031b8f99ff1e56cb11ee783016970999f30c565"
checksum = "20d6062d22c272dc2e46cf4335c98ec6a1e05cb2e0fdb90808e9114ff0969a70"
dependencies = [
"indexmap 1.9.3",
"opentelemetry 0.17.0",
"opentelemetry 0.21.0",
"sysinfo",
]

@@ -3632,7 +3660,7 @@ checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113"
dependencies = [
"async-trait",
"crossbeam-channel",
"dashmap 5.5.3",
"dashmap",
"fnv",
"futures-channel",
"futures-executor",
@@ -3646,6 +3674,28 @@ dependencies = [
"tokio-stream",
]

[[package]]
name = "opentelemetry_sdk"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5"
dependencies = [
"async-trait",
"crossbeam-channel",
"futures-channel",
"futures-executor",
"futures-util",
"glob",
"once_cell",
"opentelemetry 0.21.0",
"ordered-float 4.1.1",
"percent-encoding",
"rand",
"thiserror",
"tokio",
"tokio-stream",
]

[[package]]
name = "option-ext"
version = "0.2.0"
@@ -3670,6 +3720,15 @@ dependencies = [
"num-traits",
]

[[package]]
name = "ordered-float"
version = "4.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8"
dependencies = [
"num-traits",
]

[[package]]
name = "os_str_bytes"
version = "6.5.1"
@@ -4043,39 +4102,19 @@ dependencies = [

[[package]]
name = "prost"
version = "0.9.0"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
dependencies = [
"bytes",
"prost-derive",
]

[[package]]
name = "prost-build"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
dependencies = [
"bytes",
"heck 0.3.3",
"itertools",
"lazy_static",
"log",
"multimap",
"petgraph",
"prost",
"prost-types",
"regex",
"tempfile",
"which",
]

[[package]]
name = "prost-derive"
version = "0.9.0"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
dependencies = [
"anyhow",
"itertools",
@@ -4084,16 +4123,6 @@ dependencies = [
"syn 1.0.109",
]

[[package]]
name = "prost-types"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
dependencies = [
"bytes",
"prost",
]

[[package]]
name = "pyo3"
version = "0.20.0"
@@ -4675,6 +4704,12 @@ dependencies = [
"untrusted",
]

[[package]]
name = "rustversion"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"

[[package]]
name = "ryu"
version = "1.0.15"
@@ -5171,6 +5206,12 @@ dependencies = [
"unicode-ident",
]

[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"

[[package]]
name = "syntect"
version = "5.1.0"
@@ -5204,9 +5245,9 @@ dependencies = [

[[package]]
name = "sysinfo"
version = "0.26.9"
version = "0.29.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5"
checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys",
@@ -5250,19 +5291,6 @@ version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"

[[package]]
name = "tempfile"
version = "3.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
dependencies = [
"cfg-if 1.0.0",
"fastrand 2.0.1",
"redox_syscall 0.3.5",
"rustix 0.38.20",
"windows-sys 0.48.0",
]

[[package]]
name = "termcolor"
version = "1.3.0"
@@ -5443,20 +5471,6 @@ dependencies = [
"tokio",
]

[[package]]
name = "tokio-util"
version = "0.6.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"log",
"pin-project-lite",
"tokio",
]

[[package]]
name = "tokio-util"
version = "0.7.9"
@@ -5473,13 +5487,13 @@ dependencies = [

[[package]]
name = "tonic"
version = "0.6.2"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
dependencies = [
"async-stream",
"async-trait",
"base64 0.13.1",
"axum",
"base64 0.21.4",
"bytes",
"futures-core",
"futures-util",
@@ -5491,27 +5505,12 @@ dependencies = [
"percent-encoding",
"pin-project",
"prost",
"prost-derive",
"tokio",
"tokio-stream",
"tokio-util 0.6.10",
"tower",
"tower-layer",
"tower-service",
"tracing",
"tracing-futures",
]

[[package]]
name = "tonic-build"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
dependencies = [
"proc-macro2",
"prost-build",
"quote",
"syn 1.0.109",
]

[[package]]
@@ -5528,7 +5527,7 @@ dependencies = [
"rand",
"slab",
"tokio",
"tokio-util 0.7.9",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
@@ -5553,7 +5552,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if 1.0.0",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -5580,16 +5578,6 @@ dependencies = [
"valuable",
]

[[package]]
name = "tracing-futures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
dependencies = [
"pin-project",
"tracing",
]

[[package]]
name = "tracing-log"
version = "0.1.3"
@@ -5754,6 +5742,12 @@ dependencies = [
"percent-encoding",
]

[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"

[[package]]
name = "utf8parse"
version = "0.2.1"


+ 3
- 6
binaries/runtime/Cargo.toml View File

@@ -15,11 +15,8 @@ dora-operator-api-types = { workspace = true }
dora-core = { workspace = true }
dora-tracing = { workspace = true, optional = true }
dora-metrics = { workspace = true, optional = true }
opentelemetry = { version = "0.18.0", features = [
"rt-tokio",
"metrics",
], optional = true }
opentelemetry-system-metrics = { version = "0.1.1", optional = true }
opentelemetry = { version = "0.21.0", features = ["metrics"], optional = true }
opentelemetry-system-metrics = { version = "0.1.5", optional = true }
eyre = "0.6.8"
futures = "0.3.21"
futures-concurrency = "7.1.0"
@@ -41,7 +38,7 @@ arrow = { workspace = true, features = ["ffi"] }
aligned-vec = "0.5.0"

[features]
default = ["tracing"]
default = ["tracing", "metrics"]
tracing = ["dora-tracing"]
telemetry = ["tracing", "opentelemetry", "tracing-opentelemetry"]
metrics = ["opentelemetry", "opentelemetry-system-metrics", "dora-metrics"]


+ 6
- 3
binaries/runtime/src/lib.rs View File

@@ -14,6 +14,7 @@ use operator::{run_operator, OperatorEvent, StopReason};
#[cfg(feature = "tracing")]
use dora_tracing::set_up_tracing;
use std::{
borrow::Cow,
collections::{BTreeMap, BTreeSet, HashMap},
mem,
};
@@ -123,12 +124,14 @@ async fn run(
) -> eyre::Result<()> {
#[cfg(feature = "metrics")]
let _started = {
use dora_metrics::init_meter;
use dora_metrics::init_metrics;
use opentelemetry::global;
use opentelemetry_system_metrics::init_process_observer;

let _started = init_meter();
let meter = global::meter(Box::leak(node.id().to_string().into_boxed_str()));
let _started = init_metrics().context("Could not create opentelemetry meter")?;
let meter = global::meter(Cow::Borrowed(Box::leak(
config.node_id.to_string().into_boxed_str(),
)));
init_process_observer(meter);
_started
};


+ 3
- 3
libraries/extensions/telemetry/metrics/Cargo.toml View File

@@ -10,7 +10,7 @@ license.workspace = true

[dependencies]
futures = "0.3.21"
opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics"] }
opentelemetry-otlp = { version = "0.10", features = ["tonic", "metrics"] }
opentelemetry-system-metrics = "0.1.1"
opentelemetry = { version = "0.21", features = ["metrics"] }
opentelemetry-otlp = { version = "0.14.0", features = ["tonic", "metrics"] }
tokio = { version = "1.24.2", features = ["full"] }
opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio", "metrics"] }

+ 11
- 15
libraries/extensions/telemetry/metrics/src/lib.rs View File

@@ -10,33 +10,29 @@
//! [`sysinfo`]: https://github.com/GuillaumeGomez/sysinfo
//! [`opentelemetry-rust`]: https://github.com/open-telemetry/opentelemetry-rust

use futures::stream::Stream;
use futures::StreamExt;
use opentelemetry::sdk::metrics::{selectors, PushController};
use opentelemetry_otlp::{ExportConfig, WithExportConfig};
use std::time::Duration;

// Skip first immediate tick from tokio, not needed for async_std.
fn delayed_interval(duration: Duration) -> impl Stream<Item = tokio::time::Instant> {
opentelemetry::util::tokio_interval_stream(duration).skip(1)
}
use opentelemetry::metrics::{self};
use opentelemetry_sdk::{metrics::MeterProvider, runtime};

use opentelemetry_otlp::{ExportConfig, WithExportConfig};
/// Init opentelemetry meter
///
/// Use the default Opentelemetry exporter with default config
/// TODO: Make Opentelemetry configurable
///
pub fn init_meter() -> PushController {
let export_config = ExportConfig::default();
pub fn init_metrics() -> metrics::Result<MeterProvider> {
let endpoint = std::env::var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:4317".to_string());
let export_config = ExportConfig {
endpoint,
..ExportConfig::default()
};

opentelemetry_otlp::new_pipeline()
.metrics(tokio::spawn, delayed_interval)
.metrics(runtime::Tokio)
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.with_export_config(export_config),
)
.with_aggregator_selector(selectors::simple::Selector::Exact)
.build()
.unwrap()
}

Write Preview
Loading…
Cancel
Save