diff --git a/src/sentry/feedback/usecases/ingest/create_feedback.py b/src/sentry/feedback/usecases/ingest/create_feedback.py index eb8fff3a997998..0d8c143ccd382f 100644 --- a/src/sentry/feedback/usecases/ingest/create_feedback.py +++ b/src/sentry/feedback/usecases/ingest/create_feedback.py @@ -158,7 +158,7 @@ def fix_for_issue_platform(event_data: dict[str, Any]) -> dict[str, Any]: for [k, v] in tags: tags_dict[k] = v else: - tags_dict = tags + tags_dict = tags.copy() # Avoid mutating the original event. ret_event["tags"] = tags_dict # Set the event message to the feedback message. diff --git a/tests/sentry/feedback/__init__.py b/tests/sentry/feedback/__init__.py index 7e3b70641da7cc..ebafa3b7e6ca18 100644 --- a/tests/sentry/feedback/__init__.py +++ b/tests/sentry/feedback/__init__.py @@ -1,14 +1,17 @@ -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from sentry.utils import json def mock_feedback_event( - project_id: int, dt: datetime | None = None, message: str | None = None + project_id: int, + dt: datetime | None = None, + message: str | None = None, + tags: dict[str, Any] | None = None, ) -> dict[str, Any]: if dt is None: - dt = datetime.now(UTC) + dt = datetime.now(UTC) - timedelta(minutes=5) return { "project_id": project_id, @@ -40,6 +43,7 @@ def mock_feedback_event( "url": "https://sentry.sentry.io/feedback/?statsPeriod=14d", }, }, + "tags": tags or {}, "breadcrumbs": [], "platform": "javascript", } diff --git a/tests/sentry/feedback/endpoints/test_organization_feedback_categories.py b/tests/sentry/feedback/endpoints/test_organization_feedback_categories.py index 5e133e3bd71ee3..76661bee9b1d93 100644 --- a/tests/sentry/feedback/endpoints/test_organization_feedback_categories.py +++ b/tests/sentry/feedback/endpoints/test_organization_feedback_categories.py @@ -1,37 +1,26 @@ -from typing import Any, TypedDict +from datetime import datetime from unittest.mock import patch -import requests from django.urls import reverse -from sentry.feedback.endpoints.organization_feedback_categories import MAX_GROUP_LABELS +from sentry.feedback.lib.utils import FeedbackCreationSource +from sentry.feedback.usecases.ingest.create_feedback import create_feedback_issue from sentry.feedback.usecases.label_generation import AI_LABEL_TAG_PREFIX -from sentry.issues.grouptype import FeedbackGroup -from sentry.testutils.cases import APITestCase, SnubaTestCase -from sentry.testutils.helpers.datetime import before_now +from sentry.models.project import Project +from sentry.testutils.cases import APITestCase from sentry.testutils.silo import region_silo_test -from tests.sentry.feedback import MockSeerResponse -from tests.sentry.issues.test_utils import SearchIssueTestMixin - - -class FeedbackData(TypedDict): - fingerprint: str - tags: list[tuple[str, str]] - contexts: dict[str, Any] +from tests.sentry.feedback import MockSeerResponse, mock_feedback_event @region_silo_test -class OrganizationFeedbackCategoriesTest(APITestCase, SnubaTestCase, SearchIssueTestMixin): +class OrganizationFeedbackCategoriesTest(APITestCase): endpoint = "sentry-api-0-organization-user-feedback-categories" def setUp(self) -> None: super().setUp() self.login_as(user=self.user) - self.org = self.create_organization(owner=self.user) - self.team = self.create_team( - organization=self.org, name="Sentaur Squad", members=[self.user] - ) - self.project1 = self.create_project(teams=[self.team]) + self.org = self.organization + self.project1 = self.project self.project2 = self.create_project(teams=[self.team]) self.features = { "organizations:user-feedback-ai-categorization-features": True, @@ -44,115 +33,46 @@ def setUp(self) -> None: "sentry.feedback.endpoints.organization_feedback_categories.has_seer_access", return_value=True, ) - self.mock_has_seer_access = self.mock_has_seer_access_patcher.start() + self.mock_make_signed_seer_api_request_patcher = patch( + "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" + ) + self.mock_threshold_to_get_associated_labels_patcher = patch( + "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", + 1, + ) + self.mock_min_feedbacks_context_patcher = patch( + "sentry.feedback.endpoints.organization_feedback_categories.MIN_FEEDBACKS_CONTEXT", 1 + ) - self._create_standard_feedbacks(self.project1.id) + self.mock_make_signed_seer_api_request = ( + self.mock_make_signed_seer_api_request_patcher.start() + ) + self.mock_has_seer_access = self.mock_has_seer_access_patcher.start() + self.mock_threshold_to_get_associated_labels_patcher.start() + self.mock_min_feedbacks_context_patcher.start() def tearDown(self) -> None: self.mock_has_seer_access_patcher.stop() + self.mock_make_signed_seer_api_request_patcher.stop() + self.mock_threshold_to_get_associated_labels_patcher.stop() + self.mock_min_feedbacks_context_patcher.stop() super().tearDown() - def _create_standard_feedbacks(self, project_id: int) -> None: - """Create a standard set of feedbacks for testing.""" - insert_time = before_now(hours=12) - - feedback_data: list[FeedbackData] = [ - { - "fingerprint": "feedback-1", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface")], - "contexts": {"feedback": {"message": "The UI is too slow and confusing"}}, - }, - { - "fingerprint": "feedback-2", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface")], - "contexts": {"feedback": {"message": "Button colors are hard to see"}}, - }, - { - "fingerprint": "feedback-3", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Usability"), - ], - "contexts": {"feedback": {"message": "The interface design is poor"}}, - }, - { - "fingerprint": "feedback-4", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Performance"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Speed"), - ], - "contexts": {"feedback": {"message": "Page load times are too slow"}}, - }, - { - "fingerprint": "feedback-5", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Performance"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Loading"), - ], - "contexts": {"feedback": {"message": "The app crashes frequently"}}, - }, - { - "fingerprint": "feedback-6", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Authentication"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Security"), - ], - "contexts": {"feedback": {"message": "Login doesn't work properly"}}, - }, - { - "fingerprint": "feedback-7", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "Authentication")], - "contexts": {"feedback": {"message": "Password reset is broken"}}, - }, - { - "fingerprint": "feedback-8", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Performance"), - ], - "contexts": {"feedback": {"message": "The interface is slow and confusing"}}, - }, - { - "fingerprint": "feedback-9", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Performance"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "User Interface"), - ], - "contexts": {"feedback": {"message": "Performance issues with the UI"}}, - }, - { - "fingerprint": "feedback-10", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Authentication"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "User Interface"), - ], - "contexts": {"feedback": {"message": "Authentication problems with slow UI"}}, - }, - ] - - for data in feedback_data: - self.store_search_issue( - project_id=project_id, - user_id=1, - fingerprints=[data["fingerprint"]], - tags=data["tags"], - event_data={"contexts": data["contexts"]}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=insert_time, - ) - - # Create additional feedbacks in project2 - insert_time = before_now(hours=12) - for i in range(5): - self.store_search_issue( - project_id=self.project2.id, - user_id=1, - fingerprints=[f"feedback-project2-{i}"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface")], - event_data={"contexts": {"feedback": {"message": f"Feedback {i} from project2"}}}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=insert_time, - ) + def _create_feedback( + self, + message: str, + labels: list[str], + project: Project, + dt: datetime | None = None, + ) -> None: + tags = {f"{AI_LABEL_TAG_PREFIX}.label.{i}": labels[i] for i in range(len(labels))} + event = mock_feedback_event( + project.id, + message=message, + tags=tags, + dt=dt, + ) + create_feedback_issue(event, project, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE) def test_get_feedback_categories_without_feature_flag(self) -> None: response = self.get_error_response(self.org.slug) @@ -164,15 +84,13 @@ def test_get_feedback_categories_without_seer_access(self) -> None: response = self.get_error_response(self.org.slug) assert response.status_code == 403 - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_get_feedback_categories_basic(self, mock_seer_api_request) -> None: - mock_response = MockSeerResponse( + def test_get_feedback_categories_basic(self) -> None: + self._create_feedback("a", ["User Interface", "Speed"], self.project1) + self._create_feedback("b", ["Performance", "Usability", "Loading"], self.project1) + self._create_feedback("c", ["Security", "Performance"], self.project2) + self._create_feedback("d", ["Performance", "User Interface", "Speed"], self.project2) + + self.mock_make_signed_seer_api_request.return_value = MockSeerResponse( 200, json_data={ "data": [ @@ -181,262 +99,141 @@ def test_get_feedback_categories_basic(self, mock_seer_api_request) -> None: "associatedLabels": ["Usability"], }, {"primaryLabel": "Performance", "associatedLabels": ["Speed", "Loading"]}, - {"primaryLabel": "Authentication", "associatedLabels": ["Security"]}, + {"primaryLabel": "Security", "associatedLabels": []}, + {"primaryLabel": "hallucinated", "associatedLabels": []}, ] }, ) - mock_seer_api_request.return_value = mock_response with self.feature(self.features): response = self.get_success_response(self.org.slug) assert response.data["success"] is True - assert "categories" in response.data - assert "numFeedbacksContext" in response.data - assert response.data["numFeedbacksContext"] == 15 + assert response.data["numFeedbacksContext"] == 4 categories = response.data["categories"] - assert len(categories) == 3 + assert len(categories) == 4 assert any(category["primaryLabel"] == "User Interface" for category in categories) assert any(category["primaryLabel"] == "Performance" for category in categories) - assert any(category["primaryLabel"] == "Authentication" for category in categories) + assert any(category["primaryLabel"] == "Security" for category in categories) + assert any(category["primaryLabel"] == "hallucinated" for category in categories) for category in categories: - assert "primaryLabel" in category - assert "associatedLabels" in category - assert "feedbackCount" in category - assert isinstance(category["primaryLabel"], str) - assert isinstance(category["associatedLabels"], list) - assert isinstance(category["feedbackCount"], int) - if category["primaryLabel"] == "User Interface": - assert category["feedbackCount"] == 11 + assert category["feedbackCount"] == 3 elif category["primaryLabel"] == "Performance": assert category["feedbackCount"] == 4 - elif category["primaryLabel"] == "Authentication": - assert category["feedbackCount"] == 3 - - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_get_feedback_categories_with_project_filter(self, mock_seer_api_request) -> None: - mock_response = MockSeerResponse( + elif category["primaryLabel"] == "Security": + assert category["feedbackCount"] == 1 + elif category["primaryLabel"] == "hallucinated": + assert category["feedbackCount"] == 0 + + def test_get_feedback_categories_with_project_filter(self) -> None: + self._create_feedback("a", ["User Interface", "Performance"], self.project1) + self._create_feedback("b", ["Performance", "Loading"], self.project1) + self._create_feedback("c", ["Security", "Performance"], self.project2) + self._create_feedback("d", ["Performance", "User Interface", "Speed"], self.project2) + + self.mock_make_signed_seer_api_request.return_value = MockSeerResponse( 200, json_data={ "data": [ { "primaryLabel": "User Interface", - "associatedLabels": ["Performance", "Usability"], + "associatedLabels": [], }, - {"primaryLabel": "Authentication", "associatedLabels": ["Security", "Login"]}, + {"primaryLabel": "Performance", "associatedLabels": ["Loading"]}, ] }, ) - mock_seer_api_request.return_value = mock_response - - params = { - "project": [self.project1.id], - } with self.feature(self.features): - response = self.get_success_response(self.org.slug, **params) + response = self.get_success_response(self.org.slug, project=[self.project1.id]) assert response.data["success"] is True - assert "categories" in response.data - assert "numFeedbacksContext" in response.data - assert response.data["numFeedbacksContext"] == 10 + assert response.data["numFeedbacksContext"] == 2 categories = response.data["categories"] assert len(categories) == 2 assert any(category["primaryLabel"] == "User Interface" for category in categories) - assert any(category["primaryLabel"] == "Authentication" for category in categories) + assert any(category["primaryLabel"] == "Performance" for category in categories) for category in categories: - assert "primaryLabel" in category - assert "associatedLabels" in category - assert "feedbackCount" in category - assert isinstance(category["primaryLabel"], str) - assert isinstance(category["associatedLabels"], list) - assert isinstance(category["feedbackCount"], int) - if category["primaryLabel"] == "User Interface": - assert category["feedbackCount"] == 8 - elif category["primaryLabel"] == "Authentication": - assert category["feedbackCount"] == 3 + assert category["feedbackCount"] == 1 + elif category["primaryLabel"] == "Performance": + assert category["feedbackCount"] == 2 @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" + "sentry.feedback.endpoints.organization_feedback_categories.MAX_GROUP_LABELS", + 2, ) - def test_max_group_labels_limit(self, mock_seer_api_request) -> None: + def test_max_group_labels_limit(self) -> None: """Test that MAX_GROUP_LABELS constant is respected when processing label groups.""" - # Mock Seer to return a label group with more than MAX_GROUP_LABELS associated labels - - # Since the primary label is included in the label group, we need to have exactly MAX_GROUP_LABELS - 1 associated labels that get included - new_labels = [f"Label{i}" for i in range(MAX_GROUP_LABELS - 1)] + [ - "Extra Label 1", - "Extra Label 2", - "Extra Label 3", - ] - insert_time = before_now(hours=12) - for i in range(len(new_labels)): - self.store_search_issue( - project_id=self.project1.id, - user_id=1, - fingerprints=[f"feedback-project1-{i}"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", new_labels[i])], - event_data={ - "contexts": {"feedback": {"message": f"New feedback {i} from project1"}} - }, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=insert_time, - ) + self._create_feedback("a", ["User Interface"], self.project1) + self._create_feedback("b", ["User Interface", "Usability"], self.project1) + self._create_feedback("c", ["Accessibility"], self.project1) - mock_response = MockSeerResponse( + # Mock Seer to return a label group with more than MAX_GROUP_LABELS labels + self.mock_make_signed_seer_api_request.return_value = MockSeerResponse( 200, json_data={ "data": [ { "primaryLabel": "User Interface", - "associatedLabels": new_labels, + "associatedLabels": ["Usability", "Accessibility"], } ] }, ) - mock_seer_api_request.return_value = mock_response with self.feature(self.features): response = self.get_success_response(self.org.slug) assert response.data["success"] is True - assert "categories" in response.data - categories = response.data["categories"] assert len(categories) == 1 - user_interface_category = categories[0] - primary_label = user_interface_category["primaryLabel"] - assert primary_label == "User Interface" - - # Verify that the total number of labels in the label group is truncated to MAX_GROUP_LABELS - associated_labels = user_interface_category["associatedLabels"] - assert ( - len(associated_labels) == MAX_GROUP_LABELS - 1 - ), f"Expected {MAX_GROUP_LABELS - 1} associated labels, got {len(associated_labels)}" - - # Verify the first MAX_GROUP_LABELS labels are preserved (the primary label and the associated labels) - assert associated_labels == new_labels[: MAX_GROUP_LABELS - 1] - - # Verify that the extra labels beyond MAX_GROUP_LABELS are not included - assert "Extra Label 1" not in associated_labels - assert "Extra Label 2" not in associated_labels - assert "Extra Label 3" not in associated_labels + assert categories[0]["primaryLabel"] == "User Interface" + # Assert associated labels were truncated to length (MAX_GROUP_LABELS - 1) + assert categories[0]["associatedLabels"] == ["Usability"] - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_filter_invalid_associated_labels_by_count_ratio(self, mock_seer_api_request) -> None: + def test_filter_invalid_associated_labels_by_count_ratio(self) -> None: """Test that associated labels with too many feedbacks (relative to primary label) are filtered out.""" - # Create feedbacks where "Usability" has more feedbacks than "Navigation" (the primary label) - # This should cause "Usability" to be filtered out as an invalid associated label - - # Create feedbacks for "Navigation" (primary label) - 4 feedbacks - for i in range(4): - self.store_search_issue( - project_id=self.project1.id, - user_id=1, - fingerprints=[f"navigation-feedback-{i}"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", "Navigation")], - event_data={"contexts": {"feedback": {"message": f"Navigation issue {i}"}}}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=before_now(hours=12), - ) - - # Create feedbacks for "Usability" - 5 feedbacks (more than Navigation) - for i in range(5): - self.store_search_issue( - project_id=self.project1.id, - user_id=1, - fingerprints=[f"usability-feedback-{i}"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", "Usability")], - event_data={"contexts": {"feedback": {"message": f"Usability issue {i}"}}}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=before_now(hours=12), - ) + # Create feedbacks where associated label feedbacks are >= primary label feedbacks. + # This should cause them to be filtered out from the label group. + self._create_feedback("a", ["User Interface", "Issues UI"], self.project1) + self._create_feedback("b", ["Usability", "Issues UI"], self.project1) - self.store_search_issue( - project_id=self.project1.id, - user_id=1, - fingerprints=["design-feedback-0"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", "Design")], - event_data={"contexts": {"feedback": {"message": "Design issue 0"}}}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=before_now(hours=12), - ) + # XXX: the endpoint checks for assoc >= 3/4 * primary, but this test is more lenient in case the ratio changes. - # Mock Seer to return "Navigation" as primary with "Usability" and "Design" as associated - # "Usability" should be filtered out because it has 5 feedbacks > 3/4 * 4 = 3 - # "Design" should be kept because it has 1 feedbacks <= 3/4 * 4 = 3 - mock_response = MockSeerResponse( + self.mock_make_signed_seer_api_request.return_value = MockSeerResponse( 200, json_data={ "data": [ - {"primaryLabel": "Navigation", "associatedLabels": ["Usability", "Design"]} + { + "primaryLabel": "User Interface", + "associatedLabels": ["Usability", "Issues UI"], + } ] }, ) - mock_seer_api_request.return_value = mock_response with self.feature(self.features): response = self.get_success_response(self.org.slug) assert response.data["success"] is True - assert "categories" in response.data - categories = response.data["categories"] assert len(categories) == 1 + assert categories[0]["primaryLabel"] == "User Interface" + assert categories[0]["associatedLabels"] == [] + assert categories[0]["feedbackCount"] == 1 - navigation_category = categories[0] - assert navigation_category["primaryLabel"] == "Navigation" - - # Verify that "Usability" was filtered out (too many feedbacks) - associated_labels = navigation_category["associatedLabels"] - assert ( - "Usability" not in associated_labels - ), "Usability should be filtered out (too many feedbacks)" - - # Verify that "Design" was kept (fewer feedbacks) - assert "Design" in associated_labels, "Design should be kept (fewer feedbacks)" - - # Verify the feedback counts - assert navigation_category["feedbackCount"] == 5 - - # Verify that only valid associated labels remain - assert len(associated_labels) == 1 - assert associated_labels == ["Design"] - - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_seer_timeout(self, mock_seer_api_request) -> None: - mock_seer_api_request.side_effect = requests.exceptions.Timeout("Request timed out") + def test_seer_request_error(self) -> None: + self._create_feedback("a", ["User Interface", "Issues UI"], self.project1) + self.mock_make_signed_seer_api_request.side_effect = Exception("seer failed") with self.feature(self.features): response = self.get_error_response(self.org.slug) @@ -444,51 +241,12 @@ def test_seer_timeout(self, mock_seer_api_request) -> None: assert response.status_code == 500 assert response.data["detail"] == "Failed to generate user feedback label groups" - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_seer_connection_error(self, mock_seer_api_request) -> None: - mock_seer_api_request.side_effect = requests.exceptions.ConnectionError("Connection error") - - with self.feature(self.features): - response = self.get_error_response(self.org.slug) - - assert response.status_code == 500 - assert response.data["detail"] == "Failed to generate user feedback label groups" - - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_seer_request_error(self, mock_seer_api_request) -> None: - mock_seer_api_request.side_effect = requests.exceptions.RequestException( - "Generic request error" - ) - - with self.feature(self.features): - response = self.get_error_response(self.org.slug) - - assert response.status_code == 500 - assert response.data["detail"] == "Failed to generate user feedback label groups" - - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", - 1, - ) - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_seer_http_errors(self, mock_seer_api_request) -> None: + def test_seer_http_errors(self) -> None: + self._create_feedback("a", ["User Interface", "Issues UI"], self.project1) for status in [400, 401, 403, 404, 429, 500, 502, 503, 504]: - mock_response = MockSeerResponse(status=status, json_data={"error": "Test error"}) - mock_seer_api_request.return_value = mock_response + self.mock_make_signed_seer_api_request.return_value = MockSeerResponse( + status=status, json_data={"detail": "seer failed"} + ) with self.feature(self.features): response = self.get_error_response(self.org.slug) @@ -496,54 +254,27 @@ def test_seer_http_errors(self, mock_seer_api_request) -> None: assert response.status_code == 500 assert response.data["detail"] == "Failed to generate user feedback label groups" - @patch( - "sentry.feedback.endpoints.organization_feedback_categories.make_signed_seer_api_request" - ) - def test_fallback_to_primary_labels_when_below_threshold(self, mock_seer_api_request) -> None: - """Test that when feedback count is below THRESHOLD_TO_GET_ASSOCIATED_LABELS, we fall back to primary labels only.""" - # There are definitely less feedbacks than the threshold - # Create an extra feedback for Usability, thus making it the fourth-most-frequent label - self.store_search_issue( - project_id=self.project1.id, - user_id=1, - fingerprints=["feedback-project1-extra"], - tags=[(f"{AI_LABEL_TAG_PREFIX}.label.0", "Usability")], - event_data={"contexts": {"feedback": {"message": "Extra feedback"}}}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - insert_time=before_now(hours=12), - ) + def test_fallback_to_primary_labels_when_below_threshold(self) -> None: + """Test that when feedback count is below THRESHOLD_TO_GET_ASSOCIATED_LABELS, we fall back to primary labels only (no Seer request).""" - # Mock Seer to return associated labels (but these should be ignored) - mock_response = MockSeerResponse( - 200, - json_data={ - "data": [ - { - "primaryLabel": "User Interface", - "associatedLabels": ["Usability"], - }, - {"primaryLabel": "Performance", "associatedLabels": ["Speed", "Loading"]}, - {"primaryLabel": "Authentication", "associatedLabels": ["Security"]}, - {"primaryLabel": "Usability", "associatedLabels": ["Loading"]}, - ] - }, - ) - mock_seer_api_request.return_value = mock_response + with patch( + "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS", + 2, + ): + self._create_feedback("a", ["User Interface", "Usability"], self.project1) - # Test WITHOUT the patch - use the default threshold - with self.feature(self.features): - response = self.get_success_response(self.org.slug) + with self.feature(self.features): + response = self.get_success_response(self.org.slug) - assert response.data["success"] is True - assert "categories" in response.data + assert self.mock_make_signed_seer_api_request.call_count == 0 - categories = response.data["categories"] - assert len(categories) == 4 + assert response.data["success"] is True + categories = response.data["categories"] + assert len(categories) == 2 - assert any(category["primaryLabel"] == "User Interface" for category in categories) - assert any(category["primaryLabel"] == "Performance" for category in categories) - assert any(category["primaryLabel"] == "Authentication" for category in categories) - assert any(category["primaryLabel"] == "Usability" for category in categories) + assert any(category["primaryLabel"] == "User Interface" for category in categories) + assert any(category["primaryLabel"] == "Usability" for category in categories) - for category in categories: - assert category["associatedLabels"] == [] + for category in categories: + assert category["associatedLabels"] == [] + assert category["feedbackCount"] == 1 diff --git a/tests/sentry/feedback/lib/test_feedback_query.py b/tests/sentry/feedback/lib/test_feedback_query.py deleted file mode 100644 index c6fcb25514763e..00000000000000 --- a/tests/sentry/feedback/lib/test_feedback_query.py +++ /dev/null @@ -1,353 +0,0 @@ -from typing import Any, TypedDict - -import pytest -from snuba_sdk import Column, Condition, Entity, Op, Query, Request - -from sentry.feedback.lib.label_query import ( - _get_ai_labels_from_tags, - query_label_group_counts, - query_recent_feedbacks_with_ai_labels, - query_top_ai_labels_by_feedback_count, -) -from sentry.feedback.usecases.label_generation import AI_LABEL_TAG_PREFIX -from sentry.issues.grouptype import FeedbackGroup -from sentry.snuba.dataset import Dataset -from sentry.testutils.cases import APITestCase, SnubaTestCase -from sentry.testutils.helpers.datetime import before_now -from sentry.testutils.pytest.fixtures import django_db_all -from sentry.utils.snuba import raw_snql_query -from tests.sentry.issues.test_utils import SearchIssueTestMixin - - -class FeedbackData(TypedDict): - fingerprint: str - tags: list[tuple[str, str]] - contexts: dict[str, Any] - - -@django_db_all -class TestFeedbackQuery(APITestCase, SnubaTestCase, SearchIssueTestMixin): - def setUp(self) -> None: - super().setUp() - self.project = self.create_project() - self.organization = self.project.organization - self._create_standard_feedbacks() - - def _create_standard_feedbacks(self) -> None: - """Create a standard set of feedbacks for all tests to use.""" - feedback_data: list[FeedbackData] = [ - { - "fingerprint": "feedback-1", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface")], - "contexts": {"feedback": {"message": "The UI is too slow and confusing"}}, - }, - { - "fingerprint": "feedback-2", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "Performance")], - "contexts": { - "feedback": {"message": "The app crashes frequently when loading data"} - }, - }, - { - "fingerprint": "feedback-3", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "Authentication"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Security"), - (f"{AI_LABEL_TAG_PREFIX}.label.2", "User Interface"), - ], - "contexts": { - "feedback": {"message": "Login doesn't work properly and feels insecure"} - }, - }, - { - "fingerprint": "feedback-4", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface")], - "contexts": { - "feedback": { - "message": "Button colors are hard to see and need better contrast" - } - }, - }, - { - "fingerprint": "feedback-5", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "Performance")], - "contexts": {"feedback": {"message": "Page load times are too slow"}}, - }, - { - "fingerprint": "feedback-6", - "tags": [ - (f"{AI_LABEL_TAG_PREFIX}.label.0", "User Interface"), - (f"{AI_LABEL_TAG_PREFIX}.label.1", "Performance"), - ], - "contexts": { - "feedback": {"message": "The interface is slow and the design is confusing"} - }, - }, - { - "fingerprint": "feedback-7", - "tags": [(f"{AI_LABEL_TAG_PREFIX}.label.0", "Authentication")], - "contexts": {"feedback": {"message": "Password reset functionality is broken"}}, - }, - ] - - for data in feedback_data: - self.store_search_issue( - project_id=self.project.id, - user_id=1, - fingerprints=[data["fingerprint"]], - tags=data["tags"], - event_data={"contexts": data["contexts"]}, - override_occurrence_data={"type": FeedbackGroup.type_id}, - ) - - def test_get_ai_labels_from_tags_retrieves_labels_correctly(self) -> None: - # Create a query using the function to retrieve AI labels - query = Query( - match=Entity(Dataset.IssuePlatform.value), - select=[ - _get_ai_labels_from_tags(alias="labels"), - ], - where=[ - Condition(Column("project_id"), Op.EQ, self.project.id), - Condition(Column("timestamp"), Op.GTE, before_now(days=1)), - Condition(Column("timestamp"), Op.LT, before_now(minutes=-1)), - Condition(Column("occurrence_type_id"), Op.EQ, FeedbackGroup.type_id), - ], - ) - - result = raw_snql_query( - Request( - dataset=Dataset.IssuePlatform.value, - app_id="feedback-backend-web", - query=query, - tenant_ids={"organization_id": self.organization.id}, - ), - referrer="feedbacks.label_query", - ) - - assert len(result["data"]) == 7 - all_labels = set() - for row in result["data"]: - all_labels.update(row["labels"]) - - expected_labels = { - "User Interface", - "Performance", - "Authentication", - "Security", - } - assert all_labels == expected_labels - - def test_query_top_ai_labels_by_feedback_count(self) -> None: - result = query_top_ai_labels_by_feedback_count( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(days=-1), - limit=5, - ) - - assert len(result) == 4 - - assert result[0]["label"] == "User Interface" - assert result[0]["count"] == 4 - - assert result[1]["label"] == "Performance" - assert result[1]["count"] == 3 - - assert result[2]["label"] == "Authentication" - assert result[2]["count"] == 2 - - assert result[3]["label"] == "Security" - assert result[3]["count"] == 1 - - # Test with limit=1 should return only the top label - result_single = query_top_ai_labels_by_feedback_count( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(days=-1), - limit=1, - ) - - assert len(result_single) == 1 - assert result_single[0]["label"] == "User Interface" - assert result_single[0]["count"] == 4 - - # Query with no feedbacks in time range should return empty - result_empty = query_top_ai_labels_by_feedback_count( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=30), - end=before_now(days=29), - limit=5, - ) - - assert len(result_empty) == 0 - - # Query with non-existent project should return empty - result_no_project = query_top_ai_labels_by_feedback_count( - organization_id=self.organization.id, - project_ids=[self.project.id + 1], # Non-existent project - start=before_now(days=1), - end=before_now(minutes=-1), - limit=5, - ) - - assert len(result_no_project) == 0 - - def test_query_recent_feedbacks_with_ai_labels(self) -> None: - result = query_recent_feedbacks_with_ai_labels( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(minutes=-1), - limit=10, - ) - - assert len(result) == 7 - - # Verify that each feedback has labels and a feedback message - for feedback in result: - assert "labels" in feedback - assert "feedback" in feedback - assert isinstance(feedback["labels"], list) - assert isinstance(feedback["feedback"], str) - assert len(feedback["labels"]) > 0 - - assert { - "feedback": "The UI is too slow and confusing", - "labels": ["User Interface"], - } in result - assert { - "feedback": "The app crashes frequently when loading data", - "labels": ["Performance"], - } in result - assert { - "feedback": "Login doesn't work properly and feels insecure", - "labels": ["Authentication", "Security", "User Interface"], - } in result - assert { - "feedback": "Button colors are hard to see and need better contrast", - "labels": ["User Interface"], - } in result - assert { - "feedback": "Page load times are too slow", - "labels": ["Performance"], - } in result - assert { - "feedback": "The interface is slow and the design is confusing", - "labels": ["User Interface", "Performance"], - } in result - assert { - "feedback": "Password reset functionality is broken", - "labels": ["Authentication"], - } in result - - # Query with limit=2 should return only the 2 most recent feedbacks - result_limited = query_recent_feedbacks_with_ai_labels( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(minutes=-1), - limit=2, - ) - - assert len(result_limited) == 2 - - # Query with no feedbacks in time range should return empty - result_empty = query_recent_feedbacks_with_ai_labels( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=30), - end=before_now(days=29), - limit=10, - ) - - assert len(result_empty) == 0 - - # Query with non-existent project should return empty - result_no_project = query_recent_feedbacks_with_ai_labels( - organization_id=self.organization.id, - project_ids=[self.project.id + 1], # Non-existent project - start=before_now(days=1), - end=before_now(minutes=-1), - limit=10, - ) - - assert len(result_no_project) == 0 - - def test_query_label_group_counts(self) -> None: - label_groups = [ - ["User Interface", "Performance"], - ["Authentication", "Security"], - ["User Interface"], - ] - - # Query for feedback counts by label groups - result = query_label_group_counts( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(minutes=-1), - labels_groups=label_groups, - ) - - assert len(result) == 3 - - # Group 0: ["User Interface", "Performance"] - assert result[0] == 6 - # Group 1: ["Authentication", "Security"] - assert result[1] == 2 - # Group 2: ["User Interface"] - assert result[2] == 4 - - # Empty label groups should throw a ValueError - with pytest.raises(ValueError): - query_label_group_counts( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(minutes=-1), - labels_groups=[], - ) - - # Label groups with no matching feedbacks - no_match_result = query_label_group_counts( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=1), - end=before_now(minutes=-1), - labels_groups=[["NonExistentLabel"]], - ) - - assert len(no_match_result) == 1 - assert no_match_result[0] == 0 - - # Query with no feedbacks in time range should return 0 - empty_time_result = query_label_group_counts( - organization_id=self.organization.id, - project_ids=[self.project.id], - start=before_now(days=30), - end=before_now(days=29), - labels_groups=label_groups, - ) - - assert len(empty_time_result) == 3 - assert empty_time_result[0] == 0 - assert empty_time_result[1] == 0 - assert empty_time_result[2] == 0 - - # Query with non-existent project should return 0 - no_project_result = query_label_group_counts( - organization_id=self.organization.id, - project_ids=[self.project.id + 1], # Non-existent project - start=before_now(days=1), - end=before_now(minutes=-1), - labels_groups=label_groups, - ) - - assert len(no_project_result) == 3 - assert no_project_result[0] == 0 - assert no_project_result[1] == 0 - assert no_project_result[2] == 0 diff --git a/tests/sentry/feedback/lib/test_label_query.py b/tests/sentry/feedback/lib/test_label_query.py new file mode 100644 index 00000000000000..b4ab9e1d6335b3 --- /dev/null +++ b/tests/sentry/feedback/lib/test_label_query.py @@ -0,0 +1,180 @@ +from datetime import datetime + +import pytest +from snuba_sdk import Column, Condition, Direction, Entity, Op, OrderBy, Query, Request + +from sentry.feedback.lib.label_query import ( + _get_ai_labels_from_tags, + query_label_group_counts, + query_recent_feedbacks_with_ai_labels, + query_top_ai_labels_by_feedback_count, +) +from sentry.feedback.lib.utils import FeedbackCreationSource +from sentry.feedback.usecases.ingest.create_feedback import create_feedback_issue +from sentry.feedback.usecases.label_generation import AI_LABEL_TAG_PREFIX +from sentry.issues.grouptype import FeedbackGroup +from sentry.snuba.dataset import Dataset +from sentry.testutils.cases import APITestCase +from sentry.testutils.helpers.datetime import before_now +from sentry.utils.snuba import raw_snql_query +from tests.sentry.feedback import mock_feedback_event + + +class TestLabelQuery(APITestCase): + def setUp(self) -> None: + super().setUp() + self.project = self.create_project() + self.organization = self.project.organization + + def _create_feedback(self, message: str, labels: list[str], dt: datetime | None = None) -> None: + tags = {f"{AI_LABEL_TAG_PREFIX}.label.{i}": labels[i] for i in range(len(labels))} + event = mock_feedback_event( + self.project.id, + message=message, + tags=tags, + dt=dt, + ) + create_feedback_issue(event, self.project, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE) + + def test_get_ai_labels_from_tags_retrieves_labels_correctly(self) -> None: + self._create_feedback( + "a", + ["Authentication"], + dt=before_now(days=2), + ) + self._create_feedback( + "b", + ["Authentication", "Security"], + dt=before_now(days=1), + ) + + query = Query( + match=Entity(Dataset.IssuePlatform.value), + select=[ + _get_ai_labels_from_tags(alias="labels"), + ], + where=[ + Condition(Column("project_id"), Op.EQ, self.project.id), + Condition(Column("timestamp"), Op.GTE, before_now(days=30)), + Condition(Column("timestamp"), Op.LT, before_now(days=0)), + Condition(Column("occurrence_type_id"), Op.EQ, FeedbackGroup.type_id), + ], + orderby=[OrderBy(Column("timestamp"), Direction.ASC)], + ) + + result = raw_snql_query( + Request( + dataset=Dataset.IssuePlatform.value, + app_id="feedback-backend-web", + query=query, + tenant_ids={"organization_id": self.organization.id}, + ), + referrer="feedbacks.label_query", + ) + + assert len(result["data"]) == 2 + assert {label for label in result["data"][0]["labels"]} == {"Authentication"} + assert {label for label in result["data"][1]["labels"]} == {"Authentication", "Security"} + + def test_query_top_ai_labels_by_feedback_count(self) -> None: + self._create_feedback( + "UI issue 1", + ["User Interface", "Performance"], + ) + self._create_feedback( + "UI issue 2", + ["Checkout", "User Interface"], + ) + self._create_feedback( + "UI issue 3", + ["Performance", "User Interface", "Colors"], + ) + + result = query_top_ai_labels_by_feedback_count( + organization_id=self.organization.id, + project_ids=[self.project.id], + start=before_now(days=1), + end=before_now(days=0), + limit=3, + ) + + assert len(result) == 3 + + assert result[0]["label"] == "User Interface" + assert result[0]["count"] == 3 + + assert result[1]["label"] == "Performance" + assert result[1]["count"] == 2 + + assert result[2]["label"] == "Checkout" or result[2]["label"] == "Colors" + assert result[2]["count"] == 1 + + def test_query_recent_feedbacks_with_ai_labels(self) -> None: + self._create_feedback( + "The UI is too slow and confusing", + ["User Interface"], + dt=before_now(days=3), + ) + self._create_feedback( + "The app crashes frequently when loading data", + ["Performance"], + dt=before_now(days=2), + ) + self._create_feedback( + "Hello", + [], + dt=before_now(days=1), + ) + + result = query_recent_feedbacks_with_ai_labels( + organization_id=self.organization.id, + project_ids=[self.project.id], + start=before_now(days=30), + end=before_now(days=0), + limit=1, + ) + + assert result[0] == { + "feedback": "The app crashes frequently when loading data", + "labels": ["Performance"], + } + + def test_query_label_group_counts(self) -> None: + self._create_feedback("a", ["User Interface", "Performance"]) + self._create_feedback("b", ["Performance", "Authentication"]) + self._create_feedback("c", ["Authentication", "Security"]) + + label_groups_to_expected_result = { + ("User Interface",): 1, + ("Performance",): 2, + ("Security",): 1, + ("User Interface", "Performance"): 2, + ("Performance", "Security"): 3, + ("Authentication", "Performance", "User Interface"): 3, + ("Performance", "Authentication", "Security"): 3, + ("hello",): 0, + ("Performance", "hello"): 2, + } + + # Query for feedback counts by label groups + result = query_label_group_counts( + organization_id=self.organization.id, + project_ids=[self.project.id], + start=before_now(days=1), + end=before_now(days=0), + labels_groups=[list(g) for g in label_groups_to_expected_result], + ) + + assert len(result) == len(label_groups_to_expected_result) + for i, group in enumerate(label_groups_to_expected_result.keys()): + assert result[i] == label_groups_to_expected_result[group] + + # Empty label groups should throw a ValueError + with pytest.raises(ValueError): + query_label_group_counts( + organization_id=self.organization.id, + project_ids=[self.project.id], + start=before_now(days=1), + end=before_now(days=0), + labels_groups=[], + )