Skip to content

Commit 27d794f

Browse files
ZStriker19iunanuapaullegranddc
authored
feat(sampling): datadog sampler (#17)
# What does this PR do? This PR adds the DatadogSampler, which has the following cap # Add DatadogSampler for intelligent trace sampling in dd-trace-rs ## Summary This PR adds a comprehensive sampling implementation for the Datadog tracer in Rust. The `DatadogSampler` provides probabilistic and rule-based sampling capabilities to selectively record traces based on configurable criteria. ## Features - **Rule-based sampling**: Define sampling rules based on service name, operation name, resource name, and custom tags - **Probabilistic sampling**: Apply different sampling rates to different services and operations - **Service rate sampling**: Support for dynamic sampling rates based on service/environment combinations - **Rate limiting**: Prevent excessive trace generation with configurable rate limits - **Priority sampling**: Full support for Datadog's sampling priority mechanism - **Glob pattern matching**: Use wildcards in rule definitions for flexible matching - **Proper context propagation**: Consistent sampling decisions throughout trace hierarchies - **JSON configuration**: Configure the sampler via structured JSON input - **Initialize the sampler** with configuration and pass it to the tracer - Checks span attributes according to **Datadog semantic conversions** (will refactor these constants out in next PR) ## Implementation details The implementation consists of several cooperating components: - `DatadogSampler`: The main sampler implementing OpenTelemetry's `ShouldSample` trait - `SamplingRule`: Configurable rules for matching spans and applying sampling decisions - `RateSampler`: Deterministic sampling based on trace ID hash values - `RateLimiter`: Thread-safe limiting of sampling throughput - `GlobMatcher`: Fast pattern matching with wildcard support The sampler is fully integrated with the Datadog OpenTelemetry tracer, which can be initialized with: ```rust let sampler = DatadogSampler::new(None, None); let tracer_provider = tracer_provider_builder .with_sampler(sampler) .build(); ``` ## Testing The implementation includes a comprehensive test suite covering all aspects of the sampler's behavior, including rule matching, sampling decisions, tag generation, and configuration loading. --------- Co-authored-by: Igor Unanua <[email protected]> Co-authored-by: paullegranddc <[email protected]>
1 parent 89812fb commit 27d794f

27 files changed

+5851
-671
lines changed

Cargo.lock

Lines changed: 167 additions & 106 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

LICENSE-3rdparty.yml

Lines changed: 922 additions & 60 deletions
Large diffs are not rendered by default.

datadog-opentelemetry/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ description.workspace = true
1111
[dependencies]
1212
# Workspace dependencies
1313
dd-trace = { path = "../dd-trace" }
14+
dd-trace-sampling = { path = "../dd-trace-sampling" }
1415
dd-trace-propagation = { path = "../dd-trace-propagation", features = ["opentelemetry"] }
1516

1617
# External dependencies
@@ -31,6 +32,7 @@ assert_unordered = "0.3"
3132

3233
# Libdatadog dependencies - change to a stable version once we release
3334
datadog-trace-utils = { workspace = true, features = ["test-utils"] }
35+
dd-trace-sampling = { path = "../dd-trace-sampling" }
3436

3537
# Depend on ourselves to have APIs exposed only during tests
3638
datadog-opentelemetry = { path = ".", features = ["test-utils"] }

datadog-opentelemetry/src/lib.rs

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,17 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
mod ddtrace_transform;
5+
mod sampler;
56
mod span_exporter;
67
mod span_processor;
78
mod text_map_propagator;
89
mod trace_id;
910
mod transform;
1011

11-
use std::sync::Arc;
12+
use std::sync::{Arc, RwLock};
1213

13-
use opentelemetry_sdk::trace::SdkTracerProvider;
14+
use opentelemetry_sdk::{trace::SdkTracerProvider, Resource};
15+
use sampler::Sampler;
1416
use span_processor::{DatadogSpanProcessor, TraceRegistry};
1517
use text_map_propagator::DatadogPropagator;
1618

@@ -41,41 +43,49 @@ pub fn init_datadog(
4143
// all parameters and has an install method?
4244
tracer_provider_builder: opentelemetry_sdk::trace::TracerProviderBuilder,
4345
) -> SdkTracerProvider {
46+
let (tracer_provider, propagator) = make_tracer(config, tracer_provider_builder, None);
47+
48+
opentelemetry::global::set_text_map_propagator(propagator);
49+
opentelemetry::global::set_tracer_provider(tracer_provider.clone());
50+
tracer_provider
51+
}
52+
53+
/// Create an instance of the tracer provider
54+
fn make_tracer(
55+
config: dd_trace::Config,
56+
mut tracer_provider_builder: opentelemetry_sdk::trace::TracerProviderBuilder,
57+
resource: Option<Resource>,
58+
) -> (SdkTracerProvider, DatadogPropagator) {
4459
let registry = Arc::new(TraceRegistry::new());
60+
let resource_slot = Arc::new(RwLock::new(Resource::builder_empty().build()));
61+
let sampler = Sampler::new(&config, resource_slot.clone(), registry.clone());
62+
63+
if let Some(resource) = resource {
64+
tracer_provider_builder = tracer_provider_builder.with_resource(resource)
65+
}
4566

4667
let propagator = DatadogPropagator::new(&config, registry.clone());
47-
opentelemetry::global::set_text_map_propagator(propagator);
4868

69+
let span_processor = DatadogSpanProcessor::new(config, registry.clone(), resource_slot.clone());
4970
let tracer_provider = tracer_provider_builder
50-
.with_span_processor(DatadogSpanProcessor::new(config, registry))
71+
.with_span_processor(span_processor)
72+
.with_sampler(sampler) // Use the sampler created above
5173
.with_id_generator(trace_id::TraceidGenerator)
52-
// TODO: hookup additional components
53-
// .with_sampler(sampler)
5474
.build();
55-
opentelemetry::global::set_tracer_provider(tracer_provider.clone());
56-
tracer_provider
75+
76+
(tracer_provider, propagator)
5777
}
5878

5979
#[cfg(feature = "test-utils")]
60-
/// Create a local instance of the tracer provider
61-
pub fn make_tracer(
80+
pub fn make_test_tracer(
6281
config: dd_trace::Config,
6382
tracer_provider_builder: opentelemetry_sdk::trace::TracerProviderBuilder,
64-
) -> SdkTracerProvider {
65-
use opentelemetry::KeyValue;
66-
use opentelemetry_sdk::Resource;
67-
68-
let registry = Arc::new(TraceRegistry::new());
69-
70-
tracer_provider_builder
71-
.with_resource(
72-
Resource::builder()
73-
.with_attribute(KeyValue::new("service.name", config.service().to_string()))
74-
.build(),
75-
)
76-
.with_span_processor(DatadogSpanProcessor::new(config, registry))
77-
.with_id_generator(trace_id::TraceidGenerator)
78-
// TODO: hookup additional components
79-
// .with_sampler(sampler)
80-
.build()
83+
) -> (SdkTracerProvider, DatadogPropagator) {
84+
let resource = Resource::builder()
85+
.with_attribute(opentelemetry::KeyValue::new(
86+
"service.name",
87+
config.service().to_string(),
88+
))
89+
.build();
90+
make_tracer(config, tracer_provider_builder, Some(resource))
8191
}
Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
use dd_trace::Config;
5+
use dd_trace_sampling::DatadogSampler;
6+
use opentelemetry::trace::TraceContextExt;
7+
use opentelemetry_sdk::{trace::ShouldSample, Resource};
8+
use std::{
9+
collections::HashMap,
10+
sync::{Arc, RwLock},
11+
};
12+
13+
use crate::{
14+
span_processor::{RegisterTracePropagationResult, SamplingDecision},
15+
TraceRegistry,
16+
};
17+
18+
#[derive(Debug, Clone)]
19+
pub struct Sampler {
20+
sampler: DatadogSampler,
21+
trace_registry: Arc<TraceRegistry>,
22+
}
23+
24+
impl Sampler {
25+
pub fn new(
26+
cfg: &Config,
27+
resource: Arc<RwLock<Resource>>,
28+
trace_registry: Arc<TraceRegistry>,
29+
) -> Self {
30+
let rules = cfg
31+
.trace_sampling_rules()
32+
.iter()
33+
.map(|r| {
34+
dd_trace_sampling::SamplingRule::new(
35+
r.sample_rate,
36+
r.service.clone(),
37+
r.name.clone(),
38+
r.resource.clone(),
39+
Some(r.tags.clone()),
40+
Some(r.provenance.clone()),
41+
)
42+
})
43+
.collect::<Vec<_>>();
44+
let sampler =
45+
dd_trace_sampling::DatadogSampler::new(rules, cfg.trace_rate_limit(), resource);
46+
Self {
47+
sampler,
48+
trace_registry,
49+
}
50+
}
51+
}
52+
53+
impl ShouldSample for Sampler {
54+
fn should_sample(
55+
&self,
56+
parent_context: Option<&opentelemetry::Context>,
57+
trace_id: opentelemetry::trace::TraceId,
58+
name: &str,
59+
span_kind: &opentelemetry::trace::SpanKind,
60+
attributes: &[opentelemetry::KeyValue],
61+
_links: &[opentelemetry::trace::Link],
62+
) -> opentelemetry::trace::SamplingResult {
63+
let result = self.sampler.sample(
64+
parent_context
65+
.filter(|c| c.has_active_span())
66+
.map(|c| c.span().span_context().is_sampled()),
67+
trace_id,
68+
name,
69+
span_kind,
70+
attributes,
71+
);
72+
if let Some(trace_root_info) = &result.trace_root_info {
73+
match self.trace_registry.register_trace_propagation_data(
74+
trace_id.to_bytes(),
75+
SamplingDecision {
76+
decision: trace_root_info.sampling_priority(result.is_sampled).value(),
77+
// TODO: unify these types with decision maker with the one in the span
78+
// processor
79+
decision_maker: trace_root_info.mechanism.value() as i8,
80+
},
81+
None,
82+
// TODO(paullgdc): This is here so the injector adds the t.dm tag to
83+
// tracecontext. The injector should probably inject it from
84+
// the trace propagation data instead of tags.
85+
Some(HashMap::from_iter([(
86+
"_dd.p.dm".to_string(),
87+
format!("{}", -(trace_root_info.mechanism.value() as i32)),
88+
)])),
89+
) {
90+
RegisterTracePropagationResult::Existing(sampling_decision) => {
91+
return opentelemetry::trace::SamplingResult {
92+
decision: if sampling_decision.decision > 0 {
93+
opentelemetry::trace::SamplingDecision::RecordAndSample
94+
} else {
95+
opentelemetry::trace::SamplingDecision::RecordOnly
96+
},
97+
attributes: Vec::new(),
98+
trace_state: parent_context
99+
.map(|c| c.span().span_context().trace_state().clone())
100+
.unwrap_or_default(),
101+
}
102+
}
103+
RegisterTracePropagationResult::New => {}
104+
}
105+
}
106+
107+
opentelemetry::trace::SamplingResult {
108+
decision: result.to_otel_decision(),
109+
attributes: result.to_dd_sampling_tags(),
110+
trace_state: parent_context
111+
.map(|c| c.span().span_context().trace_state().clone())
112+
.unwrap_or_default(),
113+
}
114+
}
115+
}
116+
117+
#[cfg(test)]
118+
mod tests {
119+
use super::*;
120+
use dd_trace::configuration::SamplingRuleConfig;
121+
use opentelemetry::{
122+
trace::{SamplingDecision, SpanContext, SpanKind, TraceId, TraceState},
123+
Context, SpanId, TraceFlags,
124+
};
125+
use opentelemetry_sdk::trace::ShouldSample;
126+
use std::env;
127+
128+
#[test]
129+
fn test_create_sampler_with_sampling_rules() {
130+
// Build a fresh config to pick up the env var
131+
let mut config = Config::builder();
132+
config.set_trace_sampling_rules(vec![SamplingRuleConfig {
133+
sample_rate: 0.5,
134+
service: Some("test-service".to_string()),
135+
name: None,
136+
resource: None,
137+
tags: HashMap::new(),
138+
provenance: "customer".to_string(),
139+
}]);
140+
let config = config.build();
141+
142+
let test_resource = Arc::new(RwLock::new(Resource::builder().build()));
143+
let sampler = Sampler::new(&config, test_resource, Arc::new(TraceRegistry::new()));
144+
145+
let trace_id_bytes = [1; 16];
146+
let trace_id = TraceId::from_bytes(trace_id_bytes);
147+
148+
// Basic assertion: Check if the attributes added by the sampler are not empty,
149+
// implying some sampling logic (like adding priority tags) ran.
150+
assert!(
151+
!sampler
152+
.should_sample(None, trace_id, "test", &SpanKind::Client, &[], &[])
153+
.attributes
154+
.is_empty(),
155+
"Sampler should add attributes even if decision is complex"
156+
);
157+
158+
// Clean up environment
159+
env::remove_var("DD_TRACE_SAMPLING_RULES");
160+
}
161+
162+
#[test]
163+
fn test_create_default_sampler() {
164+
// Create a default config (no rules, no specific rate limit)
165+
let config = Config::builder().build();
166+
167+
let test_resource = Arc::new(RwLock::new(Resource::builder_empty().build()));
168+
let sampler = Sampler::new(&config, test_resource, Arc::new(TraceRegistry::new()));
169+
170+
let trace_id_bytes = [2; 16];
171+
let trace_id = TraceId::from_bytes(trace_id_bytes);
172+
173+
// Verify the default sampler behavior
174+
let result = sampler.should_sample(None, trace_id, "test", &SpanKind::Client, &[], &[]);
175+
assert_eq!(
176+
result.decision,
177+
SamplingDecision::RecordAndSample,
178+
"Default sampler should record and sample by default"
179+
);
180+
}
181+
182+
#[test]
183+
fn test_trace_state_propagation() {
184+
let config = Config::builder().build();
185+
186+
let test_resource = Arc::new(RwLock::new(Resource::builder_empty().build()));
187+
let sampler = Sampler::new(&config, test_resource, Arc::new(TraceRegistry::new()));
188+
189+
let trace_id = TraceId::from_bytes([2; 16]);
190+
let span_id = SpanId::from_bytes([3; 8]);
191+
192+
for is_sampled in [true, false] {
193+
let trace_state = TraceState::from_key_value([("test_key", "test_value")]).unwrap();
194+
let span_context = SpanContext::new(
195+
trace_id,
196+
span_id,
197+
is_sampled
198+
.then_some(TraceFlags::SAMPLED)
199+
.unwrap_or_default(),
200+
true,
201+
trace_state.clone(),
202+
);
203+
204+
// Verify the sampler with a parent context
205+
let result = sampler.should_sample(
206+
Some(&Context::new().with_remote_span_context(span_context)),
207+
trace_id,
208+
"test",
209+
&SpanKind::Client,
210+
&[],
211+
&[],
212+
);
213+
assert_eq!(
214+
result.decision,
215+
if is_sampled {
216+
SamplingDecision::RecordAndSample
217+
} else {
218+
SamplingDecision::RecordOnly
219+
},
220+
"Sampler should respect parent context sampling decision"
221+
);
222+
assert_eq!(
223+
result.trace_state.header(),
224+
"test_key=test_value",
225+
"Sampler should propagate trace state from parent context"
226+
);
227+
}
228+
}
229+
}

0 commit comments

Comments
 (0)