diff --git a/CHANGELOG.md b/CHANGELOG.md index e24b2164900..980dba6fb2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Maps `unknown_error` span status to `internal_error`. ([#5202](https://github.com/getsentry/relay/pull/5202)) - Add `otp` and `two[-_]factor` to default scrubbing rules. ([#5250](https://github.com/getsentry/relay/pull/5250)) - Add event merging logic for Playstation crashes. ([#5228](https://github.com/getsentry/relay/pull/5228)) +- Implement PII scrubbing for V2 spans. ([#5168](https://github.com/getsentry/relay/pull/5168)) **Bug Fixes**: diff --git a/relay-config/src/config.rs b/relay-config/src/config.rs index 5c1358c964b..49fbd8a5474 100644 --- a/relay-config/src/config.rs +++ b/relay-config/src/config.rs @@ -617,7 +617,7 @@ pub struct Limits { pub max_profile_size: ByteSize, /// The maximum payload size for a trace metric. pub max_trace_metric_size: ByteSize, - /// The maximum payload size for a span. + /// The maximum payload size for a log. pub max_log_size: ByteSize, /// The maximum payload size for a span. pub max_span_size: ByteSize, diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs new file mode 100644 index 00000000000..e0daf48354c --- /dev/null +++ b/relay-pii/src/eap.rs @@ -0,0 +1,2202 @@ +//! Provides the `scrub` function for scrubbing EAP items. +//! +//! This module also contains tests for scrubbing attributes. + +use relay_event_schema::processor::{ + ProcessValue, ProcessingResult, ProcessingState, ValueType, process_value, +}; +use relay_protocol::Annotated; + +use crate::{AttributeMode, PiiConfig, PiiProcessor}; + +/// Scrubs an EAP item (such as an `OurLog` or `SpanV2`) with the given PII configs. +pub fn scrub( + value_type: ValueType, + item: &mut Annotated, + advanced_rules: Option<&PiiConfig>, + legacy_rule: Option<&PiiConfig>, +) -> ProcessingResult { + let state = ProcessingState::root().enter_borrowed("", None, [value_type]); + + if let Some(config) = advanced_rules { + let mut processor = PiiProcessor::new(config.compiled()) + // For advanced rules we want to treat attributes as objects. + .attribute_mode(AttributeMode::Object); + process_value(item, &mut processor, &state)?; + } + + if let Some(config) = legacy_rule { + let mut processor = PiiProcessor::new(config.compiled()) + // For "legacy" rules we want to identify attributes with their values. + .attribute_mode(AttributeMode::ValueOnly); + process_value(item, &mut processor, &state)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use relay_event_schema::processor::ValueType; + use relay_event_schema::protocol::{Attributes, OurLog, SpanV2}; + use relay_protocol::{Annotated, SerializableAnnotated}; + + use crate::{DataScrubbingConfig, PiiConfig, eap}; + + #[test] + fn test_scrub_attributes_pii_default_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `user.name` is the only one that should be scrubbed by default rules. + let json = r#"{ + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "secret123" + }, + "secret": { + "type": "string", + "value": "topsecret" + }, + "api_key": { + "type": "string", + "value": "sk-1234567890abcdef" + }, + "auth_token": { + "type": "string", + "value": "bearer_token_123" + }, + "credit_card": { + "type": "string", + "value": "4571234567890111" + }, + "visa_card": { + "type": "string", + "value": "4571 2345 6789 0111" + }, + "bank_account": { + "type": "string", + "value": "DE89370400440532013000" + }, + "iban_code": { + "type": "string", + "value": "NO9386011117945" + }, + "authorization": { + "type": "string", + "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + }, + "private_key": { + "type": "string", + "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" + }, + "database_url": { + "type": "string", + "value": "https://username:password@example.com/db" + }, + "file_path": { + "type": "string", + "value": "C:\\Users\\john\\Documents\\secret.txt" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "social_security": { + "type": "string", + "value": "078-05-1120" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "client_ip": { + "type": "string", + "value": "192.168.1.100" + }, + "ipv6_addr": { + "type": "string", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "very_sensitive_data": { + "type": "string", + "value": "very_sensitive_data" + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let data_scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + scrub_ip_addresses: true, + exclude_fields: vec!["public_data".to_owned()], + sensitive_fields: vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ], + ..Default::default() + }; + + let scrubbing_config = data_scrubbing_config.pii_config().unwrap(); + + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data)); + } + + #[test] + fn test_scrub_attributes_pii_custom_object_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. + let json = r#" + { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: false, + scrub_ip_addresses: false, + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + }, + "project:4": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[USER NAME]" + } + }, + "project:5": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[RELEASE]" + } + }, + "project:6": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[URL PATH]" + } + }, + "project:7": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[DESCRIPTION]" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ], + "'user.name'.value": [ + "project:4" + ], + "'sentry.release'.value": [ + "project:5" + ], + "'url.path'.value": [ + "project:6" + ], + "'sentry.description'.value": [ + "project:7" + ] + } + } + )) + .unwrap(); + + eap::scrub( + ValueType::Span, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "sentry.description": { + "type": "string", + "value": "[DESCRIPTION]" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "url.path": { + "type": "string", + "value": "[URL PATH]" + }, + "user.name": { + "type": "string", + "value": "[USER NAME]" + }, + "_meta": { + "sentry.description": { + "value": { + "": { + "rem": [ + [ + "project:7", + "s", + 0, + 13 + ] + ], + "len": 9 + } + } + }, + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + }, + "url.path": { + "value": { + "": { + "rem": [ + [ + "project:6", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "project:4", + "s", + 0, + 11 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_sensitive_fields() { + let json = r#" + { + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "should_be_removed" + }, + "another_sensitive": { + "type": "integer", + "value": 42 + }, + "my_value": { + "type": "string", + "value": "this_should_be_removed_as_sensitive" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: false, + scrub_ip_addresses: false, + sensitive_fields: vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "sensitive_custom".to_owned(), + "another_sensitive".to_owned(), + ], + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "another_sensitive": { + "type": "integer", + "value": null + }, + "my_value": { + "type": "string", + "value": "[Filtered]" + }, + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "another_sensitive": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "x" + ] + ] + } + } + }, + "my_value": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 35 + } + } + }, + "sensitive_custom": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 17 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_safe_fields() { + let json = r#" + { + "password": { + "type": "string", + "value": "secret123" + }, + "credit_card": { + "type": "string", + "value": "4242424242424242" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + scrub_ip_addresses: false, + exclude_fields: vec!["secret".to_owned()], // Only 'secret' is safe + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + }, + "_meta": { + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_implicit_attribute_value() { + let json = r#" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match + } + })) + .unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + eap::scrub( + ValueType::OurLog, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + "###); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "remove_this_string_abc123.value": ["remove_abc123"], + } + })) + .unwrap(); + + eap::scrub( + ValueType::OurLog, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc---" + }, + "_meta": { + "remove_this_string_abc123": { + "value": { + "": { + "rem": [ + [ + "remove_abc123", + "s", + 0, + 6 + ] + ], + "len": 6 + } + } + } + } + } + "###); + } + + macro_rules! attribute_rule_test { + ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => { + #[test] + fn $test_name() { + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": [$rule_type] + } + })) + .unwrap(); + + let mut scrubbing_config = crate::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + let span_json = format!(r#"{{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = $rule_type, + value = $test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut span = Annotated::::from_json(&span_json).unwrap(); + + eap::scrub(ValueType::Span, &mut span, Some(&config), scrubbing_config.as_ref()).unwrap(); + + insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&span.value().unwrap().attributes), @$snapshot)); + + let log_json = format!(r#"{{ + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = $rule_type, + value = $test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut log = Annotated::::from_json(&log_json).unwrap(); + + eap::scrub(ValueType::OurLog, &mut log, Some(&config), scrubbing_config.as_ref()).unwrap(); + + insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&log.value().unwrap().attributes), @$snapshot)); + } + }; + } + + // IP rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip, "@ip", "127.0.0.1", @r###" + { + "@ip|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_replace, "@ip:replace", "127.0.0.1", @r###" + { + "@ip:replace|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip:replace|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_remove, "@ip:remove", "127.0.0.1", @r###" + { + "@ip:remove|127.0.0.1": { + "type": "string", + "value": "" + }, + "_meta": { + "@ip:remove|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:remove", + "x", + 0, + 0 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_mask, "@ip:mask", "127.0.0.1", @r###" + { + "@ip:mask|127.0.0.1": { + "type": "string", + "value": "*********" + }, + "_meta": { + "@ip:mask|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:mask", + "m", + 0, + 9 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + // Email rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email, "@email", "test@example.com", @r###" + { + "@email|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_replace, "@email:replace", "test@example.com", @r###" + { + "@email:replace|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email:replace|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:replace", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_remove, "@email:remove", "test@example.com", @r###" + { + "@email:remove|test@example.com": { + "type": "string", + "value": "" + }, + "_meta": { + "@email:remove|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_mask, "@email:mask", "test@example.com", @r###" + { + "@email:mask|test@example.com": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@email:mask|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // Credit card rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard, "@creditcard", "4242424242424242", @r###" + { + "@creditcard|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_replace, "@creditcard:replace", "4242424242424242", @r###" + { + "@creditcard:replace|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard:replace|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:replace", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_remove, "@creditcard:remove", "4242424242424242", @r###" + { + "@creditcard:remove|4242424242424242": { + "type": "string", + "value": "" + }, + "_meta": { + "@creditcard:remove|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_mask, "@creditcard:mask", "4242424242424242", @r###" + { + "@creditcard:mask|4242424242424242": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@creditcard:mask|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // IBAN rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban, "@iban", "DE89370400440532013000", @r###" + { + "@iban|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_replace, "@iban:replace", "DE89370400440532013000", @r###" + { + "@iban:replace|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban:replace|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:replace", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_remove, "@iban:remove", "DE89370400440532013000", @r###" + { + "@iban:remove|DE89370400440532013000": { + "type": "string", + "value": "" + }, + "_meta": { + "@iban:remove|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:remove", + "x", + 0, + 0 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_mask, "@iban:mask", "DE89370400440532013000", @r###" + { + "@iban:mask|DE89370400440532013000": { + "type": "string", + "value": "**********************" + }, + "_meta": { + "@iban:mask|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:mask", + "m", + 0, + 22 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + // MAC address rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac, "@mac", "4a:00:04:10:9b:50", @r###" + { + "@mac|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_replace, "@mac:replace", "4a:00:04:10:9b:50", @r###" + { + "@mac:replace|4a:00:04:10:9b:50": { + "type": "string", + "value": "[mac]" + }, + "_meta": { + "@mac:replace|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:replace", + "s", + 0, + 5 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_remove, "@mac:remove", "4a:00:04:10:9b:50", @r###" + { + "@mac:remove|4a:00:04:10:9b:50": { + "type": "string", + "value": "" + }, + "_meta": { + "@mac:remove|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:remove", + "x", + 0, + 0 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_mask, "@mac:mask", "4a:00:04:10:9b:50", @r###" + { + "@mac:mask|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac:mask|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:mask", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + // UUID rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid, "@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_replace, "@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "[uuid]" + }, + "_meta": { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:replace", + "s", + 0, + 6 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_remove, "@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "" + }, + "_meta": { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:remove", + "x", + 0, + 0 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_mask, "@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:mask", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + // IMEI rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei, "@imei", "356938035643809", @r###" + { + "@imei|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_replace, "@imei:replace", "356938035643809", @r###" + { + "@imei:replace|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei:replace|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:replace", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_remove, "@imei:remove", "356938035643809", @r###" + { + "@imei:remove|356938035643809": { + "type": "string", + "value": "" + }, + "_meta": { + "@imei:remove|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // PEM key rules + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey, + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey_replace, + "@pemkey:replace", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:replace", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey_remove, + "@pemkey:remove", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:remove", + "x", + 31, + 31 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + // URL auth rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth, "@urlauth", "https://username:password@example.com/", @r###" + { + "@urlauth|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_replace, "@urlauth:replace", "https://username:password@example.com/", @r###" + { + "@urlauth:replace|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth:replace|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:replace", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_remove, "@urlauth:remove", "https://username:password@example.com/", @r###" + { + "@urlauth:remove|https://username:password@example.com/": { + "type": "string", + "value": "https://@example.com/" + }, + "_meta": { + "@urlauth:remove|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:remove", + "x", + 8, + 8 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_mask, "@urlauth:mask", "https://username:password@example.com/", @r###" + { + "@urlauth:mask|https://username:password@example.com/": { + "type": "string", + "value": "https://*****************@example.com/" + }, + "_meta": { + "@urlauth:mask|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:mask", + "m", + 8, + 25 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + // US SSN rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn, "@usssn", "078-05-1120", @r###" + { + "@usssn|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_replace, "@usssn:replace", "078-05-1120", @r###" + { + "@usssn:replace|078-05-1120": { + "type": "string", + "value": "[us-ssn]" + }, + "_meta": { + "@usssn:replace|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:replace", + "s", + 0, + 8 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_remove, "@usssn:remove", "078-05-1120", @r###" + { + "@usssn:remove|078-05-1120": { + "type": "string", + "value": "" + }, + "_meta": { + "@usssn:remove|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:remove", + "x", + 0, + 0 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_mask, "@usssn:mask", "078-05-1120", @r###" + { + "@usssn:mask|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn:mask|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:mask", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + // User path rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath, "@userpath", "/Users/john/Documents", @r###" + { + "@userpath|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_replace, "@userpath:replace", "/Users/john/Documents", @r###" + { + "@userpath:replace|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath:replace|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:replace", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_remove, "@userpath:remove", "/Users/john/Documents", @r###" + { + "@userpath:remove|/Users/john/Documents": { + "type": "string", + "value": "/Users//Documents" + }, + "_meta": { + "@userpath:remove|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:remove", + "x", + 7, + 7 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_mask, "@userpath:mask", "/Users/john/Documents", @r###" + { + "@userpath:mask|/Users/john/Documents": { + "type": "string", + "value": "/Users/****/Documents" + }, + "_meta": { + "@userpath:mask|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:mask", + "m", + 7, + 11 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + // Password rules + // @password defaults to remove + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password, "@password", "my_password_123", @r###" + { + "@password|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_remove, "@password:remove", "my_password_123", @r###" + { + "@password:remove|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password:remove|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_replace, "@password:replace", "my_password_123", @r###" + { + "@password:replace|my_password_123": { + "type": "string", + "value": "[password]" + }, + "_meta": { + "@password:replace|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:replace", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_mask, "@password:mask", "my_password_123", @r###" + { + "@password:mask|my_password_123": { + "type": "string", + "value": "***************" + }, + "_meta": { + "@password:mask|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:mask", + "m", + 0, + 15 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // Bearer token rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_bearer, "@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", @r###" + { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_replace, + "@bearer:replace", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:replace", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_remove, + "@bearer:remove", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "" + }, + "_meta": { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:remove", + "x", + 0, + 0 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_mask, + "@bearer:mask", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "*******************************************" + }, + "_meta": { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:mask", + "m", + 0, + 43 + ] + ], + "len": 43 + } + } + } + } + } + "###); +} diff --git a/relay-pii/src/lib.rs b/relay-pii/src/lib.rs index 4d1f108c26d..efc02d70e17 100644 --- a/relay-pii/src/lib.rs +++ b/relay-pii/src/lib.rs @@ -21,6 +21,7 @@ mod regexes; mod selector; mod utils; +pub mod eap; pub mod transform; pub use self::attachments::*; diff --git a/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap b/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap new file mode 100644 index 00000000000..c959525916f --- /dev/null +++ b/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap @@ -0,0 +1,367 @@ +--- +source: relay-pii/src/eap.rs +expression: SerializableAnnotated(&data) +--- +{ + "api_key": { + "type": "string", + "value": "[Filtered]" + }, + "auth_token": { + "type": "string", + "value": "[Filtered]" + }, + "authorization": { + "type": "string", + "value": "[Filtered]" + }, + "bank_account": { + "type": "string", + "value": "[Filtered]" + }, + "client_ip": { + "type": "string", + "value": "[ip]" + }, + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "database_url": { + "type": "string", + "value": "[Filtered]" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "file_path": { + "type": "string", + "value": "[Filtered]" + }, + "iban_code": { + "type": "string", + "value": "[Filtered]" + }, + "ipv6_addr": { + "type": "string", + "value": "[ip]" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "private_key": { + "type": "string", + "value": "[Filtered]" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "secret": { + "type": "string", + "value": "[Filtered]" + }, + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "social_security": { + "type": "string", + "value": "[Filtered]" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "very_sensitive_data": { + "type": "string", + "value": "[Filtered]" + }, + "visa_card": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "api_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "auth_token": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "authorization": { + "value": { + "": { + "rem": [ + [ + "@bearer:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + }, + "bank_account": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 22 + } + } + }, + "client_ip": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 13 + } + } + }, + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "database_url": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 40 + } + } + }, + "file_path": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 34 + } + } + }, + "iban_code": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + }, + "ipv6_addr": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 39 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "private_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 118 + } + } + }, + "secret": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "social_security": { + "value": { + "": { + "rem": [ + [ + "@usssn:filter", + "s", + 0, + 10 + ] + ], + "len": 11 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "very_sensitive_data": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "visa_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/mod.rs b/relay-server/src/processing/logs/mod.rs index c65ce13e512..582066401b9 100644 --- a/relay-server/src/processing/logs/mod.rs +++ b/relay-server/src/processing/logs/mod.rs @@ -154,10 +154,11 @@ impl processing::Processor for LogsProcessor { let mut logs = process::expand(logs); process::normalize(&mut logs); filter::filter(&mut logs, ctx); - process::scrub(&mut logs, ctx); self.limiter.enforce_quotas(&mut logs, ctx).await?; + process::scrub(&mut logs, ctx); + Ok(Output::just(LogOutput::Processed(logs))) } } diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 5951db94978..22131e93809 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -1,7 +1,6 @@ use relay_event_normalization::{SchemaProcessor, eap}; use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::{OurLog, OurLogHeader}; -use relay_pii::{AttributeMode, PiiProcessor}; use relay_protocol::Annotated; use relay_quotas::DataCategory; @@ -98,21 +97,12 @@ fn scrub_log(log: &mut Annotated, ctx: Context<'_>) -> Result<()> { .pii_config() .map_err(|e| Error::PiiConfig(e.clone()))?; - let state = ProcessingState::root().enter_borrowed("", None, [ValueType::OurLog]); - - if let Some(ref config) = ctx.project_info.config.pii_config { - let mut processor = PiiProcessor::new(config.compiled()) - // For advanced rules we want to treat attributes as objects. - .attribute_mode(AttributeMode::Object); - process_value(log, &mut processor, &state)?; - } - - if let Some(config) = pii_config_from_scrubbing { - let mut processor = PiiProcessor::new(config.compiled()) - // For "legacy" rules we want to identify attributes with their values. - .attribute_mode(AttributeMode::ValueOnly); - process_value(log, &mut processor, &state)?; - } + relay_pii::eap::scrub( + ValueType::OurLog, + log, + ctx.project_info.config.pii_config.as_ref(), + pii_config_from_scrubbing.as_ref(), + )?; Ok(()) } @@ -165,910 +155,6 @@ mod tests { } } - #[test] - fn test_scrub_log_pii_default_rules() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `user.name` is the only one that should be scrubbed by default rules. - let json = r#"{ - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log message with sensitive data: user@example.com and 4571234567890111", - "attributes": { - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "secret123" - }, - "secret": { - "type": "string", - "value": "topsecret" - }, - "api_key": { - "type": "string", - "value": "sk-1234567890abcdef" - }, - "auth_token": { - "type": "string", - "value": "bearer_token_123" - }, - "credit_card": { - "type": "string", - "value": "4571234567890111" - }, - "visa_card": { - "type": "string", - "value": "4571 2345 6789 0111" - }, - "bank_account": { - "type": "string", - "value": "DE89370400440532013000" - }, - "iban_code": { - "type": "string", - "value": "NO9386011117945" - }, - "authorization": { - "type": "string", - "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" - }, - "private_key": { - "type": "string", - "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" - }, - "database_url": { - "type": "string", - "value": "https://username:password@example.com/db" - }, - "file_path": { - "type": "string", - "value": "C:\\Users\\john\\Documents\\secret.txt" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "social_security": { - "type": "string", - "value": "078-05-1120" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "client_ip": { - "type": "string", - "value": "192.168.1.100" - }, - "ipv6_addr": { - "type": "string", - "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "very_sensitive_data": { - "type": "string", - "value": "very_sensitive_data" - } - } - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = true; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "very_sensitive_data".to_owned(), - ]; - scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; - - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "api_key": { - "type": "string", - "value": "[Filtered]" - }, - "auth_token": { - "type": "string", - "value": "[Filtered]" - }, - "authorization": { - "type": "string", - "value": "[Filtered]" - }, - "bank_account": { - "type": "string", - "value": "[Filtered]" - }, - "client_ip": { - "type": "string", - "value": "[ip]" - }, - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "database_url": { - "type": "string", - "value": "[Filtered]" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "file_path": { - "type": "string", - "value": "[Filtered]" - }, - "iban_code": { - "type": "string", - "value": "[Filtered]" - }, - "ipv6_addr": { - "type": "string", - "value": "[ip]" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "private_key": { - "type": "string", - "value": "[Filtered]" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "secret": { - "type": "string", - "value": "[Filtered]" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "social_security": { - "type": "string", - "value": "[Filtered]" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "[Filtered]" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "very_sensitive_data": { - "type": "string", - "value": "[Filtered]" - }, - "visa_card": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "api_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "auth_token": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "authorization": { - "value": { - "": { - "rem": [ - [ - "@bearer:filter", - "s", - 0, - 10 - ] - ], - "len": 43 - } - } - }, - "bank_account": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 22 - } - } - }, - "client_ip": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 13 - } - } - }, - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "database_url": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 40 - } - } - }, - "file_path": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 34 - } - } - }, - "iban_code": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - }, - "ipv6_addr": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 39 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "private_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 118 - } - } - }, - "secret": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "social_security": { - "value": { - "": { - "rem": [ - [ - "@usssn:filter", - "s", - 0, - 10 - ] - ], - "len": 11 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "very_sensitive_data": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "visa_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_pii_custom_object_rules() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log message with sensitive data: user@example.com and 4571234567890111", - "attributes": { - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "test_field_mac": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "test_field_imei": { - "type": "string", - "value": "356938035643809" - }, - "test_field_uuid": { - "type": "string", - "value": "123e4567-e89b-12d3-a456-426614174000" - }, - "test_field_regex_passes": { - "type": "string", - "value": "wxyz" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - } - } - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let config = serde_json::from_value::(serde_json::json!( - { - "rules": { - "project:0": { - "type": "mac", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:1": { - "type": "imei", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:2": { - "type": "uuid", - "redaction": { - "method": "replace", - "text": "BYE" - } - }, - "project:3": { - "type": "pattern", - "pattern": "[w-z]+", - "redaction": { - "method": "replace", - "text": "REGEXED" - } - }, - "project:4": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[USER NAME]" - } - }, - "project:5": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[RELEASE]" - } - }, - "project:6": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[URL PATH]" - } - } - }, - "applications": { - "test_field_mac.value": [ - "project:0" - ], - "test_field_imei.value": [ - "project:1" - ], - "test_field_uuid.value": [ - "project:2" - ], - "test_field_regex_passes.value || test_field_regex_fails.value": [ - "project:3" - ], - "'user.name'.value": [ - "project:4" - ], - "'sentry.release'.value": [ - "project:5" - ], - "'url.path'.value": [ - "project:6" - ] - } - } - )) - .unwrap(); - - let ctx = make_context(scrubbing_config, Some(config)); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "test_field_imei": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_mac": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - }, - "test_field_regex_passes": { - "type": "string", - "value": "REGEXED" - }, - "test_field_uuid": { - "type": "string", - "value": "BYE" - }, - "url.path": { - "type": "string", - "value": "[URL PATH]" - }, - "user.name": { - "type": "string", - "value": "[USER NAME]" - }, - "_meta": { - "test_field_imei": { - "value": { - "": { - "rem": [ - [ - "project:1", - "s", - 0, - 8 - ] - ], - "len": 15 - } - } - }, - "test_field_mac": { - "value": { - "": { - "rem": [ - [ - "project:0", - "s", - 0, - 8 - ] - ], - "len": 17 - } - } - }, - "test_field_regex_passes": { - "value": { - "": { - "rem": [ - [ - "project:3", - "s", - 0, - 7 - ] - ], - "len": 4 - } - } - }, - "test_field_uuid": { - "value": { - "": { - "rem": [ - [ - "project:2", - "s", - 0, - 3 - ] - ], - "len": 36 - } - } - }, - "url.path": { - "value": { - "": { - "rem": [ - [ - "project:6", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "project:4", - "s", - 0, - 11 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_pii_string_rules() { - let test_cases = vec![ - // IP rules - ("@ip", "127.0.0.1"), - ("@ip:replace", "127.0.0.1"), - ("@ip:remove", "127.0.0.1"), - ("@ip:mask", "127.0.0.1"), - // Email rules - ("@email", "test@example.com"), - ("@email:replace", "test@example.com"), - ("@email:remove", "test@example.com"), - ("@email:mask", "test@example.com"), - // Credit card rules - ("@creditcard", "4242424242424242"), - ("@creditcard:replace", "4242424242424242"), - ("@creditcard:remove", "4242424242424242"), - ("@creditcard:mask", "4242424242424242"), - // IBAN rules - ("@iban", "DE89370400440532013000"), - ("@iban:replace", "DE89370400440532013000"), - ("@iban:remove", "DE89370400440532013000"), - ("@iban:mask", "DE89370400440532013000"), - // MAC address rules - ("@mac", "4a:00:04:10:9b:50"), - ("@mac:replace", "4a:00:04:10:9b:50"), - ("@mac:remove", "4a:00:04:10:9b:50"), - ("@mac:mask", "4a:00:04:10:9b:50"), - // UUID rules - ("@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - // IMEI rules - ("@imei", "356938035643809"), - ("@imei:replace", "356938035643809"), - ("@imei:remove", "356938035643809"), - // PEM key rules - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:replace", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:remove", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - // URL auth rules - ("@urlauth", "https://username:password@example.com/"), - ("@urlauth:replace", "https://username:password@example.com/"), - ("@urlauth:remove", "https://username:password@example.com/"), - ("@urlauth:mask", "https://username:password@example.com/"), - // US SSN rules - ("@usssn", "078-05-1120"), - ("@usssn:replace", "078-05-1120"), - ("@usssn:remove", "078-05-1120"), - ("@usssn:mask", "078-05-1120"), - // User path rules - ("@userpath", "/Users/john/Documents"), - ("@userpath:replace", "/Users/john/Documents"), - ("@userpath:remove", "/Users/john/Documents"), - ("@userpath:mask", "/Users/john/Documents"), - // Password rules - ("@password", "my_password_123"), // @password defaults to remove - ("@password:remove", "my_password_123"), - ("@password:replace", "my_password_123"), - ("@password:mask", "my_password_123"), - // Bearer token rules - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"), - ( - "@bearer:replace", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:remove", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:mask", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ]; - - for (rule_type, test_value) in test_cases { - let json = format!( - r#"{{ - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log", - "attributes": {{ - "{rule_type}|{value}": {{ - "type": "string", - "value": "{value}" - }} - }} - }}"#, - rule_type = rule_type, - value = test_value - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n") - ); - - let mut data = Annotated::::from_json(&json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "applications": { - "$string": [rule_type] - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes)); - } - } - #[test] fn test_scrub_log_base_fields() { let json = r#" @@ -1099,346 +185,6 @@ mod tests { insta::assert_json_snapshot!(SerializableAnnotated(&data)); } - #[test] - fn test_scrub_log_implicit_attribute_value_does_not_match() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$log.attributes.remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "###); - } - - #[test] - fn test_scrub_log_explicit_attribute_value_does_match() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$log.attributes.remove_this_string_abc123.value": ["remove_abc123"], - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc---" - } - }, - "_meta": { - "attributes": { - "remove_this_string_abc123": { - "value": { - "": { - "rem": [ - [ - "remove_abc123", - "s", - 0, - 6 - ] - ], - "len": 6 - } - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_sensitive_fields() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log", - "attributes": { - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "should_be_removed" - }, - "another_sensitive": { - "type": "integer", - "value": 42 - }, - "my_value": { - "type": "string", - "value": "this_should_be_removed_as_sensitive" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "sensitive_custom".to_owned(), - "another_sensitive".to_owned(), - ]; - - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "another_sensitive": { - "type": "integer", - "value": null - }, - "my_value": { - "type": "string", - "value": "[Filtered]" - }, - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "another_sensitive": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "x" - ] - ] - } - } - }, - "my_value": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 35 - } - } - }, - "sensitive_custom": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 17 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_safe_fields() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log with email@example.com", - "attributes": { - "password": { - "type": "string", - "value": "secret123" - }, - "credit_card": { - "type": "string", - "value": "4242424242424242" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.exclude_fields = vec!["secret".to_owned()]; // Only 'secret' is safe - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - }, - "_meta": { - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } - #[test] fn test_scrub_log_deep_wild_cards() { let json = r#" diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap deleted file mode 100644 index 41c9d6f79ca..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:replace|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard:replace|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:replace", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap deleted file mode 100644 index 913c5f144b6..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:remove|4242424242424242": { - "type": "string", - "value": "" - }, - "_meta": { - "@creditcard:remove|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap deleted file mode 100644 index 657f4914f6a..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:mask|4242424242424242": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@creditcard:mask|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap deleted file mode 100644 index 19a90e43cf7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap deleted file mode 100644 index 9fd82d71461..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:replace|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban:replace|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:replace", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap deleted file mode 100644 index ee757166106..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:remove|DE89370400440532013000": { - "type": "string", - "value": "" - }, - "_meta": { - "@iban:remove|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:remove", - "x", - 0, - 0 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap deleted file mode 100644 index 28fc20365c3..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:mask|DE89370400440532013000": { - "type": "string", - "value": "**********************" - }, - "_meta": { - "@iban:mask|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:mask", - "m", - 0, - 22 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap deleted file mode 100644 index a31b5045b9c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap deleted file mode 100644 index 374def85bb7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:replace|4a:00:04:10:9b:50": { - "type": "string", - "value": "[mac]" - }, - "_meta": { - "@mac:replace|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:replace", - "s", - 0, - 5 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap deleted file mode 100644 index 53d0822aacb..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:remove|4a:00:04:10:9b:50": { - "type": "string", - "value": "" - }, - "_meta": { - "@mac:remove|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:remove", - "x", - 0, - 0 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap deleted file mode 100644 index bfab00f6000..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:replace|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip:replace|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap deleted file mode 100644 index 0e54edf838d..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:mask|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac:mask|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:mask", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap deleted file mode 100644 index 9d2fdf8e1b3..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap deleted file mode 100644 index 6add3fbe4c2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "[uuid]" - }, - "_meta": { - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:replace", - "s", - 0, - 6 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap deleted file mode 100644 index ecd17f3f6a2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "" - }, - "_meta": { - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:remove", - "x", - 0, - 0 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap deleted file mode 100644 index 9dd20e89202..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:mask", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap deleted file mode 100644 index 0091511248e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap deleted file mode 100644 index f05004683e7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:replace|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei:replace|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:replace", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap deleted file mode 100644 index f207cc734da..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:remove|356938035643809": { - "type": "string", - "value": "" - }, - "_meta": { - "@imei:remove|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap deleted file mode 100644 index a547b9b429e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap deleted file mode 100644 index 23a25e68a32..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:replace", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap deleted file mode 100644 index 8a66721fcb4..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:remove|127.0.0.1": { - "type": "string", - "value": "" - }, - "_meta": { - "@ip:remove|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:remove", - "x", - 0, - 0 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap deleted file mode 100644 index dae1df1dca2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:remove", - "x", - 31, - 31 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap deleted file mode 100644 index 47d54445c74..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap deleted file mode 100644 index f5167af0386..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:replace|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth:replace|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:replace", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap deleted file mode 100644 index 4d87f406b35..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:remove|https://username:password@example.com/": { - "type": "string", - "value": "https://@example.com/" - }, - "_meta": { - "@urlauth:remove|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:remove", - "x", - 8, - 8 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap deleted file mode 100644 index 4223b7551db..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:mask|https://username:password@example.com/": { - "type": "string", - "value": "https://*****************@example.com/" - }, - "_meta": { - "@urlauth:mask|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:mask", - "m", - 8, - 25 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap deleted file mode 100644 index 2f67315fcd1..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap deleted file mode 100644 index 5e97251cb03..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:replace|078-05-1120": { - "type": "string", - "value": "[us-ssn]" - }, - "_meta": { - "@usssn:replace|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:replace", - "s", - 0, - 8 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap deleted file mode 100644 index e3a7906f097..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:remove|078-05-1120": { - "type": "string", - "value": "" - }, - "_meta": { - "@usssn:remove|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:remove", - "x", - 0, - 0 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap deleted file mode 100644 index 9623a53cea0..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:mask|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn:mask|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:mask", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap deleted file mode 100644 index a66df61b449..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap deleted file mode 100644 index ce8078e830c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:mask|127.0.0.1": { - "type": "string", - "value": "*********" - }, - "_meta": { - "@ip:mask|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:mask", - "m", - 0, - 9 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap deleted file mode 100644 index 547fd830f26..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:replace|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath:replace|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:replace", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap deleted file mode 100644 index 5b05bee508e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:remove|/Users/john/Documents": { - "type": "string", - "value": "/Users//Documents" - }, - "_meta": { - "@userpath:remove|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:remove", - "x", - 7, - 7 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap deleted file mode 100644 index 08036842544..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:mask|/Users/john/Documents": { - "type": "string", - "value": "/Users/****/Documents" - }, - "_meta": { - "@userpath:mask|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:mask", - "m", - 7, - 11 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap deleted file mode 100644 index 5747256ab36..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap deleted file mode 100644 index bfe78f0ce11..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:remove|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password:remove|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap deleted file mode 100644 index 1146e3722c8..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:replace|my_password_123": { - "type": "string", - "value": "[password]" - }, - "_meta": { - "@password:replace|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:replace", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap deleted file mode 100644 index c2534332bb1..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:mask|my_password_123": { - "type": "string", - "value": "***************" - }, - "_meta": { - "@password:mask|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:mask", - "m", - 0, - 15 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap deleted file mode 100644 index f5cac4a7f8f..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap deleted file mode 100644 index 5d31471635c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:replace", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap deleted file mode 100644 index d4c861f0031..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "" - }, - "_meta": { - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:remove", - "x", - 0, - 0 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap deleted file mode 100644 index fbb15d10c5a..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap deleted file mode 100644 index 498748ea4e4..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "*******************************************" - }, - "_meta": { - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:mask", - "m", - 0, - 43 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap deleted file mode 100644 index 4f8eafb6159..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:replace|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email:replace|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:replace", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap deleted file mode 100644 index a52fd1dce91..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:remove|test@example.com": { - "type": "string", - "value": "" - }, - "_meta": { - "@email:remove|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap deleted file mode 100644 index 6e00d96e019..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:mask|test@example.com": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@email:mask|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap deleted file mode 100644 index d16ef12fee6..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap deleted file mode 100644 index 370b5fc7527..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/mod.rs b/relay-server/src/processing/spans/mod.rs index 42848026121..3e3b4bc2d60 100644 --- a/relay-server/src/processing/spans/mod.rs +++ b/relay-server/src/processing/spans/mod.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use relay_event_normalization::GeoIpLookup; use relay_event_schema::processor::ProcessingAction; use relay_event_schema::protocol::SpanV2; +use relay_pii::PiiConfigError; use relay_quotas::{DataCategory, RateLimits}; use crate::Envelope; @@ -43,6 +44,9 @@ pub enum Error { /// A processor failed to process the spans. #[error("envelope processor failed")] ProcessingFailed(#[from] ProcessingAction), + /// Internal error, Pii config could not be loaded. + #[error("Pii configuration error")] + PiiConfig(PiiConfigError), /// The span is invalid. #[error("invalid: {0}")] Invalid(DiscardReason), @@ -60,6 +64,7 @@ impl OutcomeError for Error { let reason_code = limits.longest().and_then(|limit| limit.reason_code.clone()); Some(Outcome::RateLimited(reason_code)) } + Self::PiiConfig(_) => Some(Outcome::Invalid(DiscardReason::ProjectStatePii)), Self::ProcessingFailed(_) => Some(Outcome::Invalid(DiscardReason::Internal)), Self::Invalid(reason) => Some(Outcome::Invalid(*reason)), }; @@ -148,7 +153,7 @@ impl processing::Processor for SpansProcessor { self.limiter.enforce_quotas(&mut spans, ctx).await?; - // TODO: pii scrubbing + process::scrub(&mut spans, ctx); let metrics = dynamic_sampling::create_indexed_metrics(&spans, ctx); diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 8a6588af809..3a304de0f72 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -1,13 +1,14 @@ use relay_event_normalization::{ GeoIpLookup, SchemaProcessor, TimestampProcessor, TrimmingProcessor, eap, }; -use relay_event_schema::processor::{ProcessingState, process_value}; +use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::SpanV2; use relay_protocol::Annotated; use crate::envelope::{ContainerItems, Item, ItemContainer}; use crate::extractors::RequestMeta; use crate::managed::Managed; +use crate::processing::Context; use crate::processing::spans::{self, Error, ExpandedSpans, Result, SampledSpans}; use crate::services::outcome::DiscardReason; @@ -84,3 +85,470 @@ fn normalize_span( Ok(()) } + +/// Applies PII scrubbing to individual spans. +pub fn scrub(spans: &mut Managed, ctx: Context<'_>) { + spans.retain( + |spans| &mut spans.spans, + |span, _| { + scrub_span(span, ctx) + .inspect_err(|err| relay_log::debug!("failed to scrub pii from span: {err}")) + }, + ); +} + +fn scrub_span(span: &mut Annotated, ctx: Context<'_>) -> Result<()> { + let pii_config_from_scrubbing = ctx + .project_info + .config + .datascrubbing_settings + .pii_config() + .map_err(|e| Error::PiiConfig(e.clone()))?; + + relay_pii::eap::scrub( + ValueType::Span, + span, + ctx.project_info.config.pii_config.as_ref(), + pii_config_from_scrubbing.as_ref(), + )?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use relay_pii::{DataScrubbingConfig, PiiConfig}; + use relay_protocol::SerializableAnnotated; + + use crate::services::projects::project::ProjectInfo; + + use super::*; + + fn make_context( + scrubbing_config: DataScrubbingConfig, + pii_config: Option, + ) -> Context<'static> { + let config = Box::leak(Box::new(relay_config::Config::default())); + let global_config = Box::leak(Box::new(relay_dynamic_config::GlobalConfig::default())); + let project_info = Box::leak(Box::new(ProjectInfo { + config: relay_dynamic_config::ProjectConfig { + pii_config, + datascrubbing_settings: scrubbing_config, + ..Default::default() + }, + ..Default::default() + })); + let rate_limits = Box::leak(Box::new(relay_quotas::RateLimits::default())); + + Context { + config, + global_config, + project_info, + rate_limits, + sampling_project_info: None, + } + } + + #[test] + fn test_scrub_span_pii_default_rules_links() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `user.name` is the only one that should be scrubbed by default rules. + let json = r#"{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "links": [{ + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + } + } + }] + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = true; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ]; + scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; + + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + let link = data.value().unwrap().links.value().unwrap()[0] + .value() + .unwrap(); + insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" + { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_span_pii_custom_object_rules_links() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "links": [{ + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + } + }] + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + }, + "project:4": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[USER NAME]" + } + }, + "project:5": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[RELEASE]" + } + }, + "project:6": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[URL PATH]" + } + }, + "project:7": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[DESCRIPTION]" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ], + "'user.name'.value": [ + "project:4" + ], + "'sentry.release'.value": [ + "project:5" + ], + "'url.path'.value": [ + "project:6" + ], + "'sentry.description'.value": [ + "project:7" + ] + } + } + )) + .unwrap(); + + let ctx = make_context(scrubbing_config, Some(config)); + scrub_span(&mut data, ctx).unwrap(); + let link = data.value().unwrap().links.value().unwrap()[0] + .value() + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "sentry.description": { + "type": "string", + "value": "[DESCRIPTION]" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "url.path": { + "type": "string", + "value": "[URL PATH]" + }, + "user.name": { + "type": "string", + "value": "[USER NAME]" + }, + "_meta": { + "sentry.description": { + "value": { + "": { + "rem": [ + [ + "project:7", + "s", + 0, + 13 + ] + ], + "len": 9 + } + } + }, + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + }, + "url.path": { + "value": { + "": { + "rem": [ + [ + "project:6", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "project:4", + "s", + 0, + 11 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } +} diff --git a/relay-spans/src/v2_to_v1.rs b/relay-spans/src/v2_to_v1.rs index 401bbc6863a..fbdd74cac91 100644 --- a/relay-spans/src/v2_to_v1.rs +++ b/relay-spans/src/v2_to_v1.rs @@ -23,7 +23,6 @@ use url::Url; /// inferred from other attributes if the `sentry.op` attribute is not set. /// * The V1 span's `description` field will be set based on the V2 span's `sentry.description` /// attribute, or inferred from other attributes if the `sentry.description` attribute is not set. -/// * The V1 span's `description` field is set based on the V2 span's `sentry.description` attribute. /// * The V1 span's `status` field is set based on the V2 span's `status` field and /// `http.status_code` and `rpc.grpc.status_code` attributes. /// attribute, or the difference between the start and end timestamp if that attribute is not set. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 283a7ad9061..473b1969a82 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -18,6 +18,10 @@ get_relay_binary, latest_relay_version, ) +from .fixtures.pii import ( # noqa + scrubbing_rule, + secret_attribute, +) from .fixtures.processing import ( # noqa kafka_consumer, get_topic_name, diff --git a/tests/integration/fixtures/pii.py b/tests/integration/fixtures/pii.py new file mode 100644 index 00000000000..7927e65d711 --- /dev/null +++ b/tests/integration/fixtures/pii.py @@ -0,0 +1,61 @@ +import pytest + + +@pytest.fixture( + params=[ + ("@ip", "127.0.0.1", "[ip]"), + ("@email", "test@example.com", "[email]"), + ("@creditcard", "4242424242424242", "[creditcard]"), + ("@iban", "DE89370400440532013000", "[iban]"), + ("@mac", "4a:00:04:10:9b:50", "*****************"), + ( + "@uuid", + "ceee0822-ed8f-4622-b2a3-789e73e75cd1", + "************************************", + ), + ("@imei", "356938035643809", "[imei]"), + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", + ), + ( + "@urlauth", + "https://username:password@example.com/", + "https://[auth]@example.com/", + ), + ("@usssn", "078-05-1120", "***********"), + ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), + ("@password", "my_password_123", ""), + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), + ], + ids=[ + "ip", + "email", + "creditard", + "iban", + "mac", + "uuid", + "imei", + "pemkey", + "urlauth", + "usssn", + "userpath", + "password", + "bearer", + ], +) +def scrubbing_rule(request): + return request.param + + +@pytest.fixture( + params=[ + ("password", "my_password_123", "[Filtered]", "@password:filter"), + ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), + ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), + ], + ids=["password", "secret_key", "api_key"], +) +def secret_attribute(request): + return request.param diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index f3f2f84a8f8..dd12949260f 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -305,44 +305,12 @@ def test_ourlog_extraction_with_sentry_logs( ] -@pytest.mark.parametrize( - "rule_type,test_value,expected_scrubbed", - [ - ("@ip", "127.0.0.1", "[ip]"), - ("@email", "test@example.com", "[email]"), - ("@creditcard", "4242424242424242", "[creditcard]"), - ("@iban", "DE89370400440532013000", "[iban]"), - ("@mac", "4a:00:04:10:9b:50", "*****************"), - ( - "@uuid", - "ceee0822-ed8f-4622-b2a3-789e73e75cd1", - "************************************", - ), - ("@imei", "356938035643809", "[imei]"), - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", - ), - ( - "@urlauth", - "https://username:password@example.com/", - "https://[auth]@example.com/", - ), - ("@usssn", "078-05-1120", "***********"), - ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), - ("@password", "my_password_123", ""), - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), - ], -) def test_ourlog_extraction_with_string_pii_scrubbing( mini_sentry, relay, - items_consumer, - rule_type, - test_value, - expected_scrubbed, + scrubbing_rule, ): + rule_type, test_value, expected_scrubbed = scrubbing_rule project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["retentions"] = { @@ -375,36 +343,45 @@ def test_ourlog_extraction_with_string_pii_scrubbing( envelope = mini_sentry.captured_events.get() item_payload = json.loads(envelope.items[0].payload.bytes.decode()) item = item_payload["items"][0] - attributes = item["attributes"] - - assert "test_pii" in attributes - assert attributes["test_pii"]["value"] == expected_scrubbed - assert "_meta" in item - meta = item["_meta"]["attributes"]["test_pii"]["value"][""] - assert "rem" in meta - # Check that the rule type is mentioned in the metadata - rem_info = meta["rem"][0] - assert rule_type in rem_info[0] + assert item == { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "test_pii": {"type": "string", "value": expected_scrubbed}, + "sentry.browser.name": {"type": "string", "value": "Python Requests"}, + "sentry.browser.version": {"type": "string", "value": "2.32"}, + "sentry.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, + }, + "__header": {"byte_size": mock.ANY}, + "_meta": { + "attributes": { + "test_pii": { + "value": { + "": { + "len": mock.ANY, + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]], + } + } + } + }, + }, + "body": "Test log", + "level": "info", + "timestamp": time_within(ts), + } -@pytest.mark.parametrize( - "attribute_key,attribute_value,expected_value,rule_type", - [ - ("password", "my_password_123", "[Filtered]", "@password:filter"), - ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), - ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), - ], -) def test_ourlog_extraction_default_pii_scrubbing_attributes( mini_sentry, relay, items_consumer, - attribute_key, - attribute_value, - expected_value, - rule_type, + secret_attribute, ): + attribute_key, attribute_value, expected_value, rule_type = secret_attribute project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["features"] = [ diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index 40e207b0d9b..e1db57756f5 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -8,6 +8,7 @@ from .test_dynamic_sampling import _add_sampling_config +import json import pytest TEST_CONFIG = { @@ -607,3 +608,128 @@ def test_spans_v2_multiple_containers_not_allowed( assert mini_sentry.captured_events.empty() assert mini_sentry.captured_outcomes.empty() + + +def test_spanv2_with_string_pii_scrubbing( + mini_sentry, + relay, + scrubbing_rule, +): + rule_type, test_value, expected_scrubbed = scrubbing_rule + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:standalone-span-ingestion", + "projects:span-v2-experimental-processing", + ] + + project_config["config"]["piiConfig"]["applications"] = {"$string": [rule_type]} + + relay = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "Test span", + "is_remote": False, + "attributes": { + "test_pii": {"value": test_value, "type": "string"}, + }, + } + ) + + relay.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + + assert item == { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "test_pii": {"type": "string", "value": expected_scrubbed}, + "sentry.browser.name": {"type": "string", "value": "Python Requests"}, + "sentry.browser.version": {"type": "string", "value": "2.32"}, + "sentry.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, + }, + "_meta": { + "attributes": { + "test_pii": { + "value": { + "": { + "len": mock.ANY, + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]], + } + } + } + }, + "status": {"": {"err": ["missing_attribute"]}}, + }, + "name": "Test span", + "start_timestamp": time_within(ts), + "end_timestamp": time_within(ts.timestamp() + 0.5), + "is_remote": False, + "status": None, + } + + +def test_spanv2_default_pii_scrubbing_attributes( + mini_sentry, + relay, + secret_attribute, +): + attribute_key, attribute_value, expected_value, rule_type = secret_attribute + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:standalone-span-ingestion", + "projects:span-v2-experimental-processing", + ] + project_config["config"].setdefault( + "datascrubbingSettings", + { + "scrubData": True, + "scrubDefaults": True, + "scrubIpAddresses": True, + }, + ) + + relay_instance = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "Test span", + "attributes": { + attribute_key: {"value": attribute_value, "type": "string"}, + }, + } + ) + + relay_instance.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + attributes = item["attributes"] + + assert attribute_key in attributes + assert attributes[attribute_key]["value"] == expected_value + assert "_meta" in item + meta = item["_meta"]["attributes"][attribute_key]["value"][""] + assert "rem" in meta + rem_info = meta["rem"] + assert len(rem_info) == 1 + assert rem_info[0][0] == rule_type