feat(core): add support for dynamic tracing config

Author: NathanFlurry

Cargo.lock

@@ -280,16 +280,16 @@ path = "packages/common/error/core"
 [workspace.dependencies.rivet-error-macros]
 path = "packages/common/error/macros"
 
-[workspace.dependencies.gas]
-package = "gasoline"
-path = "packages/common/gasoline/core"
-
 [workspace.dependencies.gasoline]
 path = "packages/common/gasoline/core"
 
 [workspace.dependencies.gasoline-macros]
 path = "packages/common/gasoline/macros"
 
+[workspace.dependencies.gas]
+package = "gasoline"
+path = "packages/common/gasoline/core"
+
 [workspace.dependencies.rivet-logs]
 path = "packages/common/logs"
 
@@ -375,6 +375,9 @@ path = "packages/services/epoxy"
 [workspace.dependencies.internal]
 path = "packages/services/internal"
 
+[workspace.dependencies.rivet-tracing-reconfigure]
+path = "packages/services/tracing-reconfigure"
+
 [workspace.dependencies.namespace]
 path = "packages/services/namespace"
 

Cargo.toml

@@ -0,0 +1,78 @@
+# Dynamic Tracing Configuration
+
+Dynamically reconfigure log levels and OpenTelemetry sampling for all running services without restart.
+
+## Log Filter Configuration
+
+Control which log messages are displayed by setting filter directives (similar to `RUST_LOG`).
+
+**Set log filter to debug**
+
+```bash
+rivet-engine tracing config -f debug
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"filter":"debug"}'
+```
+
+**Debug a specific package**
+
+```bash
+rivet-engine tracing config -f "debug,rivet_api_peer=trace"
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"filter":"debug,rivet_api_peer=trace"}'
+```
+
+**Reset log filter to defaults**
+
+```bash
+rivet-engine tracing config -f ""
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"filter":null}'
+```
+
+## OpenTelemetry Sampler Ratio
+
+Control what percentage of traces are sampled and sent to the OpenTelemetry collector.
+
+**Set sampler ratio to 10%**
+
+```bash
+rivet-engine tracing config -s 0.1
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"sampler_ratio":0.1}'
+```
+
+**Set sampler ratio to 100% (capture all traces)**
+
+```bash
+rivet-engine tracing config -s 1.0
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"sampler_ratio":1.0}'
+```
+
+**Reset sampler ratio to default**
+
+```bash
+rivet-engine tracing config -s 0.001
+
+# Or via HTTP API:
+curl -X PUT http://localhost:6421/debug/tracing/config \
+  -H "Content-Type: application/json" \
+  -d '{"sampler_ratio":null}'
+```
+

dev-docs/operate/TRACING_RECONFIGURE.md

@@ -5,4 +5,4 @@ mod buckets;
 pub use buckets::*;
 pub use opentelemetry as otel;
 pub use opentelemetry::KeyValue;
-pub use providers::{OtelProviderGuard, init_otel_providers};
+pub use providers::{OtelProviderGuard, init_otel_providers, set_sampler_ratio};

out/openapi.json

@@ -1,7 +1,9 @@
 // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs
 // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs
 
+use std::sync::{Arc, RwLock, OnceLock};
 use opentelemetry::{KeyValue, global};
+use opentelemetry::trace::{SamplingResult, SpanKind};
 use opentelemetry_otlp::WithExportConfig;
 use opentelemetry_sdk::{
 	Resource,
@@ -11,6 +13,65 @@ use opentelemetry_sdk::{
 };
 use opentelemetry_semantic_conventions::{SCHEMA_URL, attribute::SERVICE_VERSION};
 
+/// Dynamic sampler that can be updated at runtime
+#[derive(Clone, Debug)]
+struct DynamicSampler {
+	ratio: Arc<RwLock<f64>>,
+}
+
+impl DynamicSampler {
+	fn new(ratio: f64) -> Self {
+		Self {
+			ratio: Arc::new(RwLock::new(ratio)),
+		}
+	}
+
+	fn set_ratio(&self, ratio: f64) {
+		if let Ok(mut r) = self.ratio.write() {
+			*r = ratio;
+		}
+	}
+}
+
+impl opentelemetry_sdk::trace::ShouldSample for DynamicSampler {
+	fn should_sample(
+		&self,
+		parent_context: Option<&opentelemetry::Context>,
+		trace_id: opentelemetry::trace::TraceId,
+		_name: &str,
+		_span_kind: &SpanKind,
+		_attributes: &[KeyValue],
+		_links: &[opentelemetry::trace::Link],
+	) -> SamplingResult {
+		let ratio = self.ratio.read().ok().map(|r| *r).unwrap_or(0.001);
+
+		// Use TraceIdRatioBased sampling logic
+		let sampler = Sampler::TraceIdRatioBased(ratio);
+		sampler.should_sample(
+			parent_context,
+			trace_id,
+			_name,
+			_span_kind,
+			_attributes,
+			_links,
+		)
+	}
+}
+
+static SAMPLER: OnceLock<DynamicSampler> = OnceLock::new();
+
+/// Update the sampler ratio at runtime
+pub fn set_sampler_ratio(ratio: f64) -> anyhow::Result<()> {
+	let sampler = SAMPLER
+		.get()
+		.ok_or_else(|| anyhow::anyhow!("sampler not initialized"))?;
+
+	sampler.set_ratio(ratio);
+	tracing::info!(?ratio, "updated sampler ratio");
+
+	Ok(())
+}
+
 fn resource() -> Resource {
 	let mut resource = Resource::builder()
 		.with_service_name(rivet_env::service_name())
@@ -48,14 +109,20 @@ fn init_tracer_provider() -> SdkTracerProvider {
 		.build()
 		.unwrap();
 
+	// Create dynamic sampler with initial ratio from env
+	let initial_ratio = std::env::var("RIVET_OTEL_SAMPLER_RATIO")
+		.ok()
+		.and_then(|s| s.parse::<f64>().ok())
+		.unwrap_or(0.001);
+
+	let dynamic_sampler = DynamicSampler::new(initial_ratio);
+
+	// Store sampler globally for later updates
+	let _ = SAMPLER.set(dynamic_sampler.clone());
+
 	SdkTracerProvider::builder()
-		// Customize sampling strategy
-		.with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
-			std::env::var("RIVET_OTEL_SAMPLER_RATIO")
-				.ok()
-				.and_then(|s| s.parse::<f64>().ok())
-				.unwrap_or(0.001),
-		))))
+		// Customize sampling strategy with parent-based sampling using our dynamic sampler
+		.with_sampler(Sampler::ParentBased(Box::new(dynamic_sampler)))
 		// If export trace to AWS X-Ray, you can use XrayIdGenerator
 		.with_id_generator(RandomIdGenerator::default())
 		.with_resource(resource())

packages/common/metrics/src/lib.rs

@@ -6,6 +6,7 @@ license.workspace = true
 edition.workspace = true
 
 [dependencies]
+anyhow.workspace = true
 console-subscriber.workspace = true
 lazy_static.workspace = true
 rivet-metrics.workspace = true

packages/common/metrics/src/providers.rs

@@ -5,6 +5,8 @@ use tokio::sync::{Notify, OnceCell};
 mod metrics;
 mod traces;
 
+pub use traces::reload_log_filter;
+
 static SHUTDOWN: OnceCell<Arc<Notify>> = OnceCell::const_new();
 
 /// Returns `None` if the runtime was shut down manually.