ref(grouping): Add training mode for similarity model rollout (#102623)

Introduce training_mode parameter to send dual embeddings during model
upgrades. Centralize model version config and add
should_send_new_model_embeddings() to track which groups need new
embeddings. Rename feature flag to be version-agnostic.

Author: yuvmen

src/sentry/event_manager.py

@@ -65,7 +65,10 @@
     run_primary_grouping,
 )
 from sentry.grouping.ingest.metrics import record_hash_calculation_metrics, record_new_group_metrics
-from sentry.grouping.ingest.seer import maybe_check_seer_for_matching_grouphash
+from sentry.grouping.ingest.seer import (
+    maybe_check_seer_for_matching_grouphash,
+    maybe_send_seer_for_new_model_training,
+)
 from sentry.grouping.ingest.utils import (
     add_group_id_to_grouphashes,
     check_for_group_creation_load_shed,
@@ -1287,6 +1290,7 @@ def assign_event_to_group(
     if primary.existing_grouphash:
         group_info = handle_existing_grouphash(job, primary.existing_grouphash, primary.grouphashes)
         result = "found_primary"
+        maybe_send_seer_for_new_model_training(event, primary.existing_grouphash, primary.variants)
     # If we haven't, try again using the secondary config. (If there is no secondary config, or
     # we're out of the transition period, we'll get back the empty `NULL_GROUPHASH_INFO`.)
     else:
@@ -1298,6 +1302,9 @@ def assign_event_to_group(
                 job, secondary.existing_grouphash, all_grouphashes
             )
             result = "found_secondary"
+            maybe_send_seer_for_new_model_training(
+                event, secondary.existing_grouphash, secondary.variants
+            )
 
         # If we still haven't found a group, ask Seer for a match (if enabled for the event's platform)
         else:

src/sentry/features/temporary.py

@@ -646,8 +646,8 @@ def register_temporary_features(manager: FeatureManager) -> None:
     manager.add("projects:similarity-embeddings", ProjectFeature, FeatureHandlerStrategy.INTERNAL, default=False, api_expose=True)
     manager.add("projects:similarity-indexing", ProjectFeature, FeatureHandlerStrategy.INTERNAL, api_expose=False)
     manager.add("projects:similarity-view", ProjectFeature, FeatureHandlerStrategy.INTERNAL, api_expose=True)
-    # Enable v2 similarity grouping model (part of v2 grouping rollout)
-    manager.add("projects:similarity-grouping-v2-model", ProjectFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    # Enable new similarity grouping model upgrade (version-agnostic rollout)
+    manager.add("projects:similarity-grouping-model-upgrade", ProjectFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     # Starfish: extract metrics from the spans
     manager.add("projects:span-metrics-extraction", ProjectFeature, FeatureHandlerStrategy.INTERNAL, api_expose=True)
     manager.add("projects:span-metrics-extraction-addons", ProjectFeature, FeatureHandlerStrategy.INTERNAL, api_expose=False)

src/sentry/grouping/ingest/seer.py

@@ -6,9 +6,8 @@
 from django.conf import settings
 from django.utils import timezone
 
-from sentry import features, options
+from sentry import options
 from sentry import ratelimits as ratelimiter
-from sentry.conf.server import SEER_SIMILARITY_MODEL_VERSION
 from sentry.grouping.grouping_info import get_grouping_info_from_variants_legacy
 from sentry.grouping.ingest.grouphash_metadata import (
     check_grouphashes_for_positive_fingerprint_match,
@@ -17,8 +16,12 @@
 from sentry.grouping.variants import BaseVariant
 from sentry.models.grouphash import GroupHash
 from sentry.models.project import Project
+from sentry.seer.similarity.config import (
+    get_grouping_model_version,
+    should_send_new_model_embeddings,
+)
 from sentry.seer.similarity.similar_issues import get_similarity_data_from_seer
-from sentry.seer.similarity.types import GroupingVersion, SimilarIssuesEmbeddingsRequest
+from sentry.seer.similarity.types import SimilarIssuesEmbeddingsRequest
 from sentry.seer.similarity.utils import (
     SEER_INELIGIBLE_EVENT_PLATFORMS,
     ReferrerOptions,
@@ -257,10 +260,17 @@ def get_seer_similar_issues(
     event: Event,
     event_grouphash: GroupHash,
     variants: dict[str, BaseVariant],
+    training_mode: bool = False,
 ) -> tuple[float | None, GroupHash | None]:
     """
     Ask Seer for the given event's nearest neighbor(s) and return the stacktrace distance and
     matching GroupHash of the closest match (if any), or `(None, None)` if no match found.
+
+    Args:
+        event: The event being grouped
+        event_grouphash: The grouphash for this event
+        variants: Grouping variants for the event
+        training_mode: If True, only possibly insert embedding without returning matches
     """
     event_hash = event.get_primary_hash()
     exception_type = get_path(event.data, "exception", "values", -1, "type")
@@ -272,10 +282,7 @@ def get_seer_similar_issues(
         get_stacktrace_string(get_grouping_info_from_variants_legacy(variants)),
     )
 
-    # Get model configuration from feature flags
-    use_v2_model = features.has("projects:similarity-grouping-v2-model", event.project)
-    model_version = GroupingVersion.V2 if use_v2_model else GroupingVersion.V1
-    training_mode = False  # PR #B will add the smart logic
+    model_version = get_grouping_model_version(event.project)
 
     request_data: SimilarIssuesEmbeddingsRequest = {
         "event_id": event.event_id,
@@ -392,12 +399,20 @@ def get_seer_similar_issues(
         "grouping.similarity.seer_results_returned",
         len(seer_results),
         sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-        tags={**metrics_tags, "is_hybrid": is_hybrid_fingerprint_case},
+        tags={
+            **metrics_tags,
+            "is_hybrid": is_hybrid_fingerprint_case,
+            "training_mode": training_mode,
+        },
     )
     metrics.incr(
         "grouping.similarity.get_seer_similar_issues",
         sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-        tags={**metrics_tags, "is_hybrid": is_hybrid_fingerprint_case},
+        tags={
+            **metrics_tags,
+            "is_hybrid": is_hybrid_fingerprint_case,
+            "training_mode": training_mode,
+        },
     )
 
     logger.info(
@@ -528,6 +543,8 @@ def maybe_check_seer_for_matching_grouphash(
 
             timestamp = timezone.now()
 
+            model_version = get_grouping_model_version(event.project)
+
             gh_metadata.update(
                 # Technically the time of the metadata record creation and the time of the Seer
                 # request will be some milliseconds apart, but a) the difference isn't meaningful
@@ -541,9 +558,55 @@ def maybe_check_seer_for_matching_grouphash(
                 date_added=gh_metadata.date_added or timestamp,
                 seer_date_sent=gh_metadata.date_added or timestamp,
                 seer_event_sent=event.event_id,
-                seer_model=SEER_SIMILARITY_MODEL_VERSION,
+                seer_model=model_version.value,
                 seer_matched_grouphash=seer_matched_grouphash,
                 seer_match_distance=seer_match_distance,
             )
 
     return seer_matched_grouphash
+
+
+@sentry_sdk.tracing.trace
+def maybe_send_seer_for_new_model_training(
+    event: Event,
+    existing_grouphash: GroupHash,
+    variants: dict[str, BaseVariant],
+) -> None:
+    """
+    Send a training_mode=true request to Seer to build embeddings for the new model
+    version if the existing grouphash hasn't been sent to the new version yet.
+
+    This only happens for projects that have the new model rolled out. It helps
+    build embeddings for existing groups without affecting production grouping decisions.
+
+    Args:
+        event: The event being grouped
+        existing_grouphash: The grouphash that was found for this event
+        variants: Grouping variants for the event
+    """
+
+    # Check if we should send embeddings for the new model
+    gh_metadata = existing_grouphash.metadata
+    grouphash_seer_model = gh_metadata.seer_model if gh_metadata else None
+
+    if not should_send_new_model_embeddings(event.project, grouphash_seer_model):
+        return
+
+    # Send training mode request (honor all checks like rate limits, circuit breaker, etc.)
+    if not should_call_seer_for_grouping(event, variants, existing_grouphash):
+        return
+
+    record_did_call_seer_metric(event, call_made=True, blocker="none", training_mode=True)
+
+    try:
+        # Call Seer with training_mode=True (results won't be used for grouping)
+        get_seer_similar_issues(event, existing_grouphash, variants, training_mode=True)
+    except Exception as e:
+        sentry_sdk.capture_exception(
+            e,
+            tags={
+                "event": event.event_id,
+                "project": event.project.id,
+                "grouphash": existing_grouphash.hash,
+            },
+        )

src/sentry/issues/endpoints/group_similar_issues_embeddings.py

@@ -6,7 +6,7 @@
 from rest_framework.request import Request
 from rest_framework.response import Response
 
-from sentry import analytics, features, options
+from sentry import analytics, options
 from sentry.api.analytics import GroupSimilarIssuesEmbeddingsCountEvent
 from sentry.api.api_owners import ApiOwner
 from sentry.api.api_publish_status import ApiPublishStatus
@@ -16,12 +16,9 @@
 from sentry.issues.endpoints.bases.group import GroupEndpoint
 from sentry.models.group import Group
 from sentry.models.grouphash import GroupHash
+from sentry.seer.similarity.config import get_grouping_model_version
 from sentry.seer.similarity.similar_issues import get_similarity_data_from_seer
-from sentry.seer.similarity.types import (
-    GroupingVersion,
-    SeerSimilarIssueData,
-    SimilarIssuesEmbeddingsRequest,
-)
+from sentry.seer.similarity.types import SeerSimilarIssueData, SimilarIssuesEmbeddingsRequest
 from sentry.seer.similarity.utils import (
     ReferrerOptions,
     event_content_has_stacktrace,
@@ -104,10 +101,7 @@ def get(self, request: Request, group: Group) -> Response:
         if not stacktrace_string or not latest_event:
             return Response([])  # No exception, stacktrace or in-app frames, or event
 
-        # Get model configuration from feature flags
-        use_v2_model = features.has("projects:similarity-grouping-v2-model", group.project)
-        model_version = GroupingVersion.V2 if use_v2_model else GroupingVersion.V1
-        training_mode = False  # TODO: currently hardcoded, follow up PR will add the logic
+        model_version = get_grouping_model_version(group.project)
 
         similar_issues_params: SimilarIssuesEmbeddingsRequest = {
             "event_id": latest_event.event_id,
@@ -119,7 +113,7 @@ def get(self, request: Request, group: Group) -> Response:
             "referrer": "similar_issues",
             "use_reranking": options.get("seer.similarity.similar_issues.use_reranking"),
             "model": model_version,
-            "training_mode": training_mode,
+            "training_mode": False,
         }
         # Add optional parameters
         if request.GET.get("k"):

src/sentry/seer/similarity/config.py

@@ -0,0 +1,99 @@
+"""
+Configuration for similarity grouping model versions.
+
+This module defines which model versions are used for similarity grouping
+and provides helper functions for determining the appropriate version to use.
+"""
+
+from sentry import features
+from sentry.models.project import Project
+from sentry.seer.similarity.types import GroupingVersion
+
+# Stable model version - used for ALL requests for non-rolled-out projects
+SEER_GROUPING_STABLE_VERSION = GroupingVersion.V1
+
+# New model version being rolled out
+# - Rolled-out projects: Use this for ALL requests (both grouping and embeddings)
+# - Non-rolled-out projects: Never use this (use stable version for everything)
+# Set to None to disable rollout entirely
+SEER_GROUPING_NEW_VERSION: GroupingVersion | None = GroupingVersion.V2
+
+# Feature flag name (version-agnostic)
+SEER_GROUPING_NEW_MODEL_ROLLOUT_FEATURE = "projects:similarity-grouping-model-upgrade"
+
+
+def get_grouping_model_version(project: Project) -> GroupingVersion:
+    """
+    Get the model version to use for grouping decisions for this project.
+
+    Returns:
+        - New version if rollout is enabled for this project
+        - Stable version otherwise
+    """
+    # Early return if no new version configured
+    if SEER_GROUPING_NEW_VERSION is None:
+        return SEER_GROUPING_STABLE_VERSION
+
+    # Type is narrowed to GroupingVersion here
+    if features.has(SEER_GROUPING_NEW_MODEL_ROLLOUT_FEATURE, project):
+        return SEER_GROUPING_NEW_VERSION
+    return SEER_GROUPING_STABLE_VERSION
+
+
+def is_new_model_rolled_out(project: Project) -> bool:
+    """
+    Check if the new model version is rolled out for this project.
+
+    Returns False if:
+    - No new version is configured (rollout disabled globally)
+    - Feature flag is not enabled for this project
+    """
+    if SEER_GROUPING_NEW_VERSION is None:
+        return False
+
+    return features.has(SEER_GROUPING_NEW_MODEL_ROLLOUT_FEATURE, project)
+
+
+def get_new_model_version() -> GroupingVersion | None:
+    """
+    Get the new model version being rolled out, if any.
+    Returns None if no rollout is in progress.
+    """
+    return SEER_GROUPING_NEW_VERSION
+
+
+def should_send_new_model_embeddings(
+    project: Project,
+    grouphash_seer_model: str | None,
+) -> bool:
+    """
+    Check if we should send training_mode=true request to build embeddings
+    for the new model version for an existing group.
+
+    This is true when:
+    1. A new version is being rolled out
+    2. The project has the rollout feature enabled
+    3. The grouphash hasn't been sent to the new version yet
+
+    Args:
+        project: The project
+        grouphash_seer_model: The seer_model value from grouphash metadata
+
+    Returns:
+        True if we should send a training_mode=true request
+    """
+    new_version = get_new_model_version()
+    if new_version is None:
+        # No rollout in progress
+        return False
+
+    if not is_new_model_rolled_out(project):
+        # Rollout not enabled for this project
+        return False
+
+    if grouphash_seer_model is None:
+        # Never sent to Seer at all
+        return True
+
+    # Check if it was sent to the new version
+    return grouphash_seer_model != new_version.value

src/sentry/seer/similarity/utils.py

@@ -314,11 +314,18 @@ def event_content_has_stacktrace(event: GroupEvent | Event) -> bool:
     return exception_stacktrace or threads_stacktrace or only_stacktrace
 
 
-def record_did_call_seer_metric(event: Event, *, call_made: bool, blocker: str) -> None:
+def record_did_call_seer_metric(
+    event: Event, *, call_made: bool, blocker: str, training_mode: bool = False
+) -> None:
     metrics.incr(
         "grouping.similarity.did_call_seer",
         sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-        tags={"call_made": call_made, "blocker": blocker, "platform": event.platform},
+        tags={
+            "call_made": call_made,
+            "blocker": blocker,
+            "platform": event.platform,
+            "training_mode": training_mode,
+        },
     )
 
 

tests/sentry/event_manager/grouping/test_seer_grouping.py

@@ -3,11 +3,10 @@
 from typing import Any
 from unittest.mock import MagicMock, patch
 
-from sentry.conf.server import SEER_SIMILARITY_MODEL_VERSION
 from sentry.grouping.ingest.grouphash_metadata import create_or_update_grouphash_metadata_if_needed
 from sentry.grouping.ingest.seer import get_seer_similar_issues, should_call_seer_for_grouping
 from sentry.models.grouphash import GroupHash
-from sentry.seer.similarity.types import SeerSimilarIssueData
+from sentry.seer.similarity.types import GroupingVersion, SeerSimilarIssueData
 from sentry.testutils.cases import TestCase
 from sentry.testutils.helpers.eventprocessing import save_new_event
 from sentry.testutils.pytest.mocking import capture_results
@@ -205,7 +204,7 @@ def test_group_with_no_seer_match(self, _: MagicMock) -> None:
                 event_grouphash,
                 event_grouphash.metadata.date_added,
                 event.event_id,
-                SEER_SIMILARITY_MODEL_VERSION,
+                GroupingVersion.V1.value,
                 None,
                 None,
             )
@@ -248,7 +247,7 @@ def test_group_with_seer_match(self, _: MagicMock) -> None:
                 new_event_grouphash,
                 new_event_grouphash.metadata.date_added,
                 new_event.event_id,
-                SEER_SIMILARITY_MODEL_VERSION,
+                GroupingVersion.V1.value,
                 existing_event_grouphash,
                 seer_result_data.stacktrace_distance,
             )