feat(replay): add feedback to LLM context (#94315)

closes
https://linear.app/getsentry/issue/REPLAY-418/add-error-and-user-feedback-context-to-llm

adds feedback context to the LLM input for AI breadcrumbs. we do this
somewhat differently from how we fetch errors for error context;
feedback breadcrumbs are readily available in the rrweb payloads, so we
can get the feedback (event) ID from the breadcrumb. from there, we
fetch the feedback event and use the info in the event (timestamp,
message) to populate the log message.

Author: michellewzhang

src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py

@@ -34,7 +34,7 @@
 logger = logging.getLogger(__name__)
 
 
-class ErrorEvent(TypedDict):
+class GroupEvent(TypedDict):
     id: str
     title: str
     message: str
@@ -97,23 +97,24 @@ def get(self, request: Request, project: Project, replay_id: str) -> Response:
             error_events = []
         else:
             error_events = fetch_error_details(project_id=project.id, error_ids=error_ids)
-
         return self.paginate(
             request=request,
             paginator_cls=GenericOffsetPaginator,
             data_fn=functools.partial(fetch_segments_metadata, project.id, replay_id),
-            on_results=functools.partial(analyze_recording_segments, error_events, replay_id),
+            on_results=functools.partial(
+                analyze_recording_segments, error_events, replay_id, project.id
+            ),
         )
 
 
-def fetch_error_details(project_id: int, error_ids: list[str]) -> list[ErrorEvent]:
-    """Fetch error details given error IDs and return a list of ErrorEvent objects."""
+def fetch_error_details(project_id: int, error_ids: list[str]) -> list[GroupEvent]:
+    """Fetch error details given error IDs and return a list of GroupEvent objects."""
     try:
         node_ids = [Event.generate_node_id(project_id, event_id=id) for id in error_ids]
         events = nodestore.backend.get_multi(node_ids)
 
         return [
-            ErrorEvent(
+            GroupEvent(
                 category="error",
                 id=event_id,
                 title=data.get("title", ""),
@@ -128,24 +129,64 @@ def fetch_error_details(project_id: int, error_ids: list[str]) -> list[ErrorEven
         return []
 
 
-def generate_error_log_message(error: ErrorEvent) -> str:
+def fetch_feedback_details(feedback_id: str | None, project_id):
+    """
+    Fetch user feedback associated with a specific feedback event ID.
+    """
+    if feedback_id is None:
+        return None
+
+    try:
+        node_id = Event.generate_node_id(project_id, event_id=feedback_id)
+        event = nodestore.backend.get(node_id)
+
+        return (
+            GroupEvent(
+                category="feedback",
+                id=feedback_id,
+                title="User Feedback",
+                timestamp=event.get("timestamp", 0.0) * 1000,  # feedback timestamp is in seconds
+                message=event.get("contexts", {}).get("feedback", {}).get("message", ""),
+            )
+            if event
+            else None
+        )
+
+    except Exception as e:
+        sentry_sdk.capture_exception(e)
+        return None
+
+
+def generate_error_log_message(error: GroupEvent) -> str:
     title = error["title"]
     message = error["message"]
     timestamp = error["timestamp"]
 
     return f"User experienced an error: '{title}: {message}' at {timestamp}"
 
 
+def generate_feedback_log_message(feedback: GroupEvent) -> str:
+    title = feedback["title"]
+    message = feedback["message"]
+    timestamp = feedback["timestamp"]
+
+    return f"User submitted feedback: '{title}: {message}' at {timestamp}"
+
+
 def get_request_data(
-    iterator: Iterator[tuple[int, memoryview]], error_events: list[ErrorEvent]
+    iterator: Iterator[tuple[int, memoryview]],
+    error_events: list[GroupEvent],
+    project_id: int,
 ) -> list[str]:
     # Sort error events by timestamp
     error_events.sort(key=lambda x: x["timestamp"])
-    return list(gen_request_data(iterator, error_events))
+    return list(gen_request_data(iterator, error_events, project_id))
 
 
 def gen_request_data(
-    iterator: Iterator[tuple[int, memoryview]], error_events: list[ErrorEvent]
+    iterator: Iterator[tuple[int, memoryview]],
+    error_events: list[GroupEvent],
+    project_id,
 ) -> Generator[str]:
     """Generate log messages from events and errors in chronological order."""
     error_idx = 0
@@ -163,7 +204,14 @@ def gen_request_data(
                 error_idx += 1
 
             # Yield the current event's log message
-            if message := as_log_message(event):
+            event_type = which(event)
+            if event_type == EventType.FEEDBACK:
+                feedback_id = event["data"]["payload"].get("data", {}).get("feedback_id", None)
+                feedback = fetch_feedback_details(feedback_id, project_id)
+                if feedback:
+                    yield generate_feedback_log_message(feedback)
+
+            elif message := as_log_message(event):
                 yield message
 
     # Yield any remaining error messages
@@ -175,12 +223,15 @@ def gen_request_data(
 
 @sentry_sdk.trace
 def analyze_recording_segments(
-    error_events: list[ErrorEvent],
+    error_events: list[GroupEvent],
     replay_id: str,
+    project_id: int,
     segments: list[RecordingSegmentStorageMeta],
 ) -> dict[str, Any]:
     # Combine breadcrumbs and error details
-    request_data = json.dumps({"logs": get_request_data(iter_segment_data(segments), error_events)})
+    request_data = json.dumps(
+        {"logs": get_request_data(iter_segment_data(segments), error_events, project_id)}
+    )
 
     # Log when the input string is too large. This is potential for timeout.
     if len(request_data) > 100000:
@@ -271,6 +322,8 @@ def as_log_message(event: dict[str, Any]) -> str | None:
             return None
         case EventType.OPTIONS:
             return None
+        case EventType.FEEDBACK:
+            return None  # the log message is processed before this method is called
 
 
 def make_seer_request(request_data: str) -> bytes:

src/sentry/replays/usecases/ingest/event_parser.py

@@ -76,6 +76,7 @@ class EventType(Enum):
     UNKNOWN = 13
     CANVAS = 14
     OPTIONS = 15
+    FEEDBACK = 16
 
 
 def which(event: dict[str, Any]) -> EventType:
@@ -135,6 +136,8 @@ def which(event: dict[str, Any]) -> EventType:
                     return EventType.HYDRATION_ERROR
                 elif category == "replay.mutations":
                     return EventType.MUTATIONS
+                elif category == "sentry.feedback":
+                    return EventType.FEEDBACK
                 else:
                     return EventType.UNKNOWN
             elif event["data"]["tag"] == "performanceSpan":

tests/sentry/replays/test_project_replay_summarize_breadcrumbs.py

@@ -1,6 +1,6 @@
 import uuid
 import zlib
-from datetime import datetime, timezone
+from datetime import UTC, datetime, timezone
 from unittest.mock import patch
 
 import requests
@@ -11,13 +11,14 @@
 from sentry import nodestore
 from sentry.eventstore.models import Event
 from sentry.replays.endpoints.project_replay_summarize_breadcrumbs import (
-    ErrorEvent,
+    GroupEvent,
     as_log_message,
     get_request_data,
 )
 from sentry.replays.lib.storage import FilestoreBlob, RecordingSegmentStorageMeta
 from sentry.replays.testutils import mock_replay
 from sentry.testutils.cases import TransactionTestCase
+from sentry.testutils.pytest.fixtures import django_db_all
 from sentry.testutils.skips import requires_snuba
 from sentry.utils import json
 
@@ -56,6 +57,42 @@ def save_recording_segment(
         )
         FilestoreBlob().set(metadata, zlib.compress(data) if compressed else data)
 
+    def mock_create_feedback_occurrence(self, project_id: int, replay_id: str | None = None):
+        dt = datetime.now(UTC)
+
+        event = {
+            "project_id": project_id,
+            "event_id": "56b08cf7852c42cbb95e4a6998c66ad6",
+            "timestamp": dt.timestamp(),
+            "received": dt.isoformat(),
+            "first_seen": dt.isoformat(),
+            "user": {
+                "ip_address": "72.164.175.154",
+                "email": "[email protected]",
+                "id": 880461,
+                "isStaff": False,
+                "name": "Josh Ferge",
+            },
+            "contexts": {
+                "feedback": {
+                    "contact_email": "[email protected]",
+                    "name": "Josh Ferge",
+                    "message": "Great website!",
+                    "replay_id": replay_id,
+                    "url": "https://sentry.sentry.io/feedback/?statsPeriod=14d",
+                },
+            },
+        }
+
+        self.store_event(
+            data={
+                "event_id": event["event_id"],
+                "timestamp": event["timestamp"],
+                "contexts": event["contexts"],
+            },
+            project_id=self.project.id,
+        )
+
     @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
     def test_get(self, make_seer_request):
         return_value = json.dumps({"hello": "world"}).encode()
@@ -215,7 +252,7 @@ def test_get_with_error(self, make_seer_request):
         assert response.content == return_value
 
     @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
-    def test_get_with_error_context_disabled(self, make_seer_request):
+    def test_get_with_error_context_disabled_and_enabled(self, make_seer_request):
         """Test handling of breadcrumbs with error context disabled"""
         return_value = json.dumps({"error": "An error happened"}).encode()
         make_seer_request.return_value = return_value
@@ -261,6 +298,7 @@ def test_get_with_error_context_disabled(self, make_seer_request):
         ]
         self.save_recording_segment(0, json.dumps(data).encode())
 
+        # with error context disabled
         with self.feature(
             {
                 "organizations:session-replay": True,
@@ -280,8 +318,88 @@ def test_get_with_error_context_disabled(self, make_seer_request):
         assert response.get("Content-Type") == "application/json"
         assert response.content == return_value
 
+        # with error context enabled
+        with self.feature(
+            {
+                "organizations:session-replay": True,
+                "organizations:replay-ai-summaries": True,
+                "organizations:gen-ai-features": True,
+            }
+        ):
+            response = self.client.get(self.url, {"enable_error_context": "true"})
+
+        call_args = json.loads(make_seer_request.call_args[0][0])
+        assert "logs" in call_args
+        assert any("ZeroDivisionError" in log for log in call_args["logs"])
+        assert any("division by zero" in log for log in call_args["logs"])
+
+        assert response.status_code == 200
+        assert response.get("Content-Type") == "application/json"
+        assert response.content == return_value
+
+    @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
+    def test_get_with_feedback(self, make_seer_request):
+        """Test handling of breadcrumbs with user feedback"""
+        return_value = json.dumps({"feedback": "Feedback was submitted"}).encode()
+        make_seer_request.return_value = return_value
+
+        self.mock_create_feedback_occurrence(self.project.id, replay_id=self.replay_id)
+
+        now = datetime.now(timezone.utc)
+
+        self.store_replays(
+            mock_replay(
+                now,
+                self.project.id,
+                self.replay_id,
+            )
+        )
+
+        data = [
+            {
+                "type": 5,
+                "timestamp": float(now.timestamp()),
+                "data": {
+                    "tag": "breadcrumb",
+                    "payload": {"category": "console", "message": "hello"},
+                },
+            },
+            {
+                "type": 5,
+                "timestamp": float(now.timestamp()),
+                "data": {
+                    "tag": "breadcrumb",
+                    "payload": {
+                        "category": "sentry.feedback",
+                        "data": {"feedback_id": "56b08cf7852c42cbb95e4a6998c66ad6"},
+                    },
+                },
+            },
+        ]
+        self.save_recording_segment(0, json.dumps(data).encode())
+
+        with self.feature(
+            {
+                "organizations:session-replay": True,
+                "organizations:replay-ai-summaries": True,
+                "organizations:gen-ai-features": True,
+            }
+        ):
+            response = self.client.get(self.url)
+
+        make_seer_request.assert_called_once()
+        call_args = json.loads(make_seer_request.call_args[0][0])
+        assert "logs" in call_args
+        assert any("Great website!" in log for log in call_args["logs"])
+        assert any("User submitted feedback" in log for log in call_args["logs"])
+
+        assert response.status_code == 200
+        assert response.get("Content-Type") == "application/json"
+        assert response.content == return_value
+
 
-def test_get_request_data():
+@django_db_all
+def test_get_request_data(default_project):
     def _faker():
         yield 0, memoryview(
             json.dumps(
@@ -307,14 +425,14 @@ def _faker():
         )
 
     error_events = [
-        ErrorEvent(
+        GroupEvent(
             category="error",
             id="123",
             title="ZeroDivisionError",
             timestamp=3.0,
             message="division by zero",
         ),
-        ErrorEvent(
+        GroupEvent(
             category="error",
             id="234",
             title="BadError",
@@ -323,7 +441,7 @@ def _faker():
         ),
     ]
 
-    result = get_request_data(_faker(), error_events=error_events)
+    result = get_request_data(_faker(), error_events=error_events, project_id=default_project.id)
     assert result == [
         "User experienced an error: 'BadError: something else bad' at 1.0",
         "Logged: hello at 1.5",

tests/sentry/replays/unit/test_event_parser.py

@@ -590,6 +590,13 @@ def test_which():
     }
     assert which(event) == EventType.CONSOLE
 
+    event = {
+        "type": 5,
+        "timestamp": 0.0,
+        "data": {"tag": "breadcrumb", "payload": {"category": "sentry.feedback"}},
+    }
+    assert which(event) == EventType.FEEDBACK
+
     event = {
         "type": 5,
         "timestamp": 0.0,