ref(similarity): Add killswitch to delete record task (#74212)

Move similarity killswitch check into utils file
Check killswitch before calling record delete functions

Author: jangjodi

src/sentry/grouping/ingest/seer.py

@@ -14,6 +14,7 @@
     event_content_is_seer_eligible,
     filter_null_from_event_title,
     get_stacktrace_string,
+    killswitch_enabled,
 )
 from sentry.utils import metrics
 from sentry.utils.safe import get_path
@@ -53,7 +54,7 @@ def should_call_seer_for_grouping(event: Event, primary_hashes: CalculatedHashes
     # and rate limiting friends instead happens in the `with_circuit_breaker` helper used where
     # `get_seer_similar_issues` is actually called. (It has to be there in order for it to track
     # errors arising from that call.)
-    if _killswitch_enabled(event, project) or _ratelimiting_enabled(event, project):
+    if killswitch_enabled(project.id, event) or _ratelimiting_enabled(event, project):
         return False
 
     return True
@@ -96,42 +97,6 @@ def _has_customized_fingerprint(event: Event, primary_hashes: CalculatedHashes)
     return False
 
 
-def _killswitch_enabled(event: Event, project: Project) -> bool:
-    """
-    Check both the global and similarity-specific Seer killswitches.
-    """
-
-    logger_extra = {"event_id": event.event_id, "project_id": project.id}
-
-    if options.get("seer.global-killswitch.enabled"):
-        logger.warning(
-            "should_call_seer_for_grouping.seer_global_killswitch_enabled",
-            extra=logger_extra,
-        )
-        metrics.incr("grouping.similarity.seer_global_killswitch_enabled")
-        metrics.incr(
-            "grouping.similarity.did_call_seer",
-            sample_rate=1.0,
-            tags={"call_made": False, "blocker": "global-killswitch"},
-        )
-        return True
-
-    if options.get("seer.similarity-killswitch.enabled"):
-        logger.warning(
-            "should_call_seer_for_grouping.seer_similarity_killswitch_enabled",
-            extra=logger_extra,
-        )
-        metrics.incr("grouping.similarity.seer_similarity_killswitch_enabled")
-        metrics.incr(
-            "grouping.similarity.did_call_seer",
-            sample_rate=1.0,
-            tags={"call_made": False, "blocker": "similarity-killswitch"},
-        )
-        return True
-
-    return False
-
-
 def _ratelimiting_enabled(event: Event, project: Project) -> bool:
     """
     Check both the global and project-based Seer similarity ratelimits.

src/sentry/seer/similarity/utils.py

@@ -1,7 +1,9 @@
 import logging
 from typing import Any, TypeVar
 
+from sentry import options
 from sentry.eventstore.models import Event
+from sentry.utils import metrics
 from sentry.utils.safe import get_path
 
 logger = logging.getLogger(__name__)
@@ -118,6 +120,42 @@ def event_content_is_seer_eligible(event: Event) -> bool:
     return True
 
 
+def killswitch_enabled(project_id: int, event: Event | None = None) -> bool:
+    """
+    Check both the global and similarity-specific Seer killswitches.
+    """
+
+    logger_extra = {"event_id": event.event_id if event else None, "project_id": project_id}
+
+    if options.get("seer.global-killswitch.enabled"):
+        logger.warning(
+            "should_call_seer_for_grouping.seer_global_killswitch_enabled",
+            extra=logger_extra,
+        )
+        metrics.incr("grouping.similarity.seer_global_killswitch_enabled")
+        metrics.incr(
+            "grouping.similarity.did_call_seer",
+            sample_rate=1.0,
+            tags={"call_made": False, "blocker": "global-killswitch"},
+        )
+        return True
+
+    if options.get("seer.similarity-killswitch.enabled"):
+        logger.warning(
+            "should_call_seer_for_grouping.seer_similarity_killswitch_enabled",
+            extra=logger_extra,
+        )
+        metrics.incr("grouping.similarity.seer_similarity_killswitch_enabled")
+        metrics.incr(
+            "grouping.similarity.did_call_seer",
+            sample_rate=1.0,
+            tags={"call_made": False, "blocker": "similarity-killswitch"},
+        )
+        return True
+
+    return False
+
+
 def filter_null_from_event_title(title: str) -> str:
     """
     Filter out null bytes from event title so that it can be saved in records table.

src/sentry/tasks/delete_seer_grouping_records.py

@@ -8,6 +8,7 @@
     delete_grouping_records_by_hash,
     delete_project_grouping_records,
 )
+from sentry.seer.similarity.utils import killswitch_enabled
 from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task
 
@@ -33,6 +34,9 @@ def delete_seer_grouping_records_by_hash(
     Task to delete seer grouping records by hash list.
     Calls the seer delete by hash endpoint with batches of hashes of size `BATCH_SIZE`.
     """
+    if killswitch_enabled(project_id):
+        return
+
     batch_size = options.get("embeddings-grouping.seer.delete-record-batch-size")
     len_hashes = len(hashes)
     end_index = min(last_deleted_index + batch_size, len_hashes)
@@ -48,7 +52,11 @@ def call_delete_seer_grouping_records_by_hash(
     if group_ids:
         group = Group.objects.get(id=group_ids[0])
         project = group.project if group else None
-    if project and features.has("projects:similarity-embeddings-delete-by-hash", project):
+    if (
+        project
+        and features.has("projects:similarity-embeddings-delete-by-hash", project)
+        and not killswitch_enabled(project.id)
+    ):
         # TODO (jangjodi): once we store seer grouping info in GroupHash, we should filter by that here
         group_hash_objects = GroupHash.objects.filter(
             project_id=project.id, group__id__in=group_ids
@@ -75,5 +83,8 @@ def call_seer_delete_project_grouping_records(
     *args: Any,
     **kwargs: Any,
 ) -> None:
+    if killswitch_enabled(project_id):
+        return
+
     logger.info("calling seer delete records by project", extra={"project_id": project_id})
     delete_project_grouping_records(project_id)