ref(ecosystem): refactor link_all_repos to bulk update repositories (#95494)

Author: cathteng

src/sentry/integrations/bitbucket/repository.py

@@ -1,9 +1,12 @@
+from typing import Any
+
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.locks import locks
 from sentry.models.apitoken import generate_token
 from sentry.models.options.organization_option import OrganizationOption
 from sentry.organizations.services.organization.model import RpcOrganization
 from sentry.plugins.providers import IntegrationRepositoryProvider
+from sentry.plugins.providers.integration_repository import RepositoryConfig
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.utils.email import parse_email, parse_user_name
 from sentry.utils.http import absolute_uri
@@ -43,7 +46,9 @@ def get_webhook_secret(self, organization):
                 )
         return secret
 
-    def build_repository_config(self, organization: RpcOrganization, data):
+    def build_repository_config(
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
         installation = self.get_installation(data.get("installation"), organization.id)
         client = installation.get_client()
         try:

src/sentry/integrations/bitbucket_server/repository.py

@@ -1,11 +1,15 @@
 from datetime import datetime, timezone
+from typing import Any
 
 from django.core.cache import cache
 from django.urls import reverse
 
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.organizations.services.organization.model import RpcOrganization
-from sentry.plugins.providers.integration_repository import IntegrationRepositoryProvider
+from sentry.plugins.providers.integration_repository import (
+    IntegrationRepositoryProvider,
+    RepositoryConfig,
+)
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.utils.hashlib import md5_text
 from sentry.utils.http import absolute_uri
@@ -30,7 +34,9 @@ def get_repository_data(self, organization, config):
             config["repo"] = repo["name"]
         return config
 
-    def build_repository_config(self, organization: RpcOrganization, data):
+    def build_repository_config(
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
         installation = self.get_installation(data.get("installation"), organization.id)
         client = installation.get_client()
 

src/sentry/integrations/github/repository.py

@@ -12,6 +12,7 @@
 from sentry.models.repository import Repository
 from sentry.organizations.services.organization.model import RpcOrganization
 from sentry.plugins.providers import IntegrationRepositoryProvider
+from sentry.plugins.providers.integration_repository import RepositoryConfig
 from sentry.shared_integrations.exceptions import ApiError, IntegrationError
 
 WEBHOOK_EVENTS = ["push", "pull_request"]
@@ -51,8 +52,8 @@ def get_repository_data(
         return config
 
     def build_repository_config(
-        self, organization: RpcOrganization, data: Mapping[str, Any]
-    ) -> Mapping[str, Any]:
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
         return {
             "name": data["identifier"],
             "external_id": data["external_id"],

src/sentry/integrations/github/tasks/link_all_repos.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 
 from sentry.constants import ObjectStatus
 from sentry.integrations.services.integration import integration_service
@@ -18,7 +19,6 @@
 from sentry.taskworker.config import TaskworkerConfig
 from sentry.taskworker.namespaces import integrations_control_tasks
 from sentry.taskworker.retry import Retry
-from sentry.utils import metrics
 
 logger = logging.getLogger(__name__)
 
@@ -58,27 +58,11 @@ def link_all_repos(
             integration_id=integration_id, status=ObjectStatus.ACTIVE
         )
         if not integration:
-            # TODO: Remove this logger in favor of context manager
-            logger.error(
-                "%s.link_all_repos.integration_missing",
-                integration_key,
-                extra={"organization_id": organization_id},
-            )
-            metrics.incr("github.link_all_repos.error", tags={"type": "missing_integration"})
             lifecycle.record_failure(str(LinkAllReposHaltReason.MISSING_INTEGRATION))
             return
 
         rpc_org = organization_service.get(id=organization_id)
         if rpc_org is None:
-            logger.error(
-                "%s.link_all_repos.organization_missing",
-                integration_key,
-                extra={"organization_id": organization_id},
-            )
-            metrics.incr(
-                f"{integration_key}.link_all_repos.error",
-                tags={"type": "missing_organization"},
-            )
             lifecycle.record_failure(str(LinkAllReposHaltReason.MISSING_ORGANIZATION))
             return
 
@@ -93,26 +77,31 @@ def link_all_repos(
                 lifecycle.record_halt(str(LinkAllReposHaltReason.RATE_LIMITED))
                 return
 
-            metrics.incr(f"{integration_key}.link_all_repos.api_error")
             raise
 
         integration_repo_provider = get_integration_repository_provider(integration)
 
-        # If we successfully create any repositories, we'll set this to True
-        success = False
-
+        repo_configs: list[dict[str, Any]] = []
+        missing_repos = []
         for repo in repositories:
             try:
-                config = get_repo_config(repo, integration_id)
-                integration_repo_provider.create_repository(
-                    repo_config=config, organization=rpc_org
-                )
-                success = True
+                repo_configs.append(get_repo_config(repo, integration_id))
             except KeyError:
-                continue
-            except RepoExistsError:
-                metrics.incr("sentry.integration_repo_provider.repo_exists")
+                missing_repos.append(repo)
                 continue
 
-        if not success:
-            lifecycle.record_halt(str(LinkAllReposHaltReason.REPOSITORY_NOT_CREATED))
+        try:
+            integration_repo_provider.create_repositories(
+                configs=repo_configs, organization=rpc_org
+            )
+        except RepoExistsError as e:
+            lifecycle.record_halt(
+                str(LinkAllReposHaltReason.REPOSITORY_NOT_CREATED),
+                {"missing_repos": e.repos, "integration_id": integration_id},
+            )
+
+        if missing_repos:
+            lifecycle.record_halt(
+                str(LinkAllReposHaltReason.REPOSITORY_NOT_CREATED),
+                {"missing_repos": missing_repos, "integration_id": integration_id},
+            )

src/sentry/integrations/github_enterprise/repository.py

@@ -1,7 +1,10 @@
+from typing import Any
+
 from sentry.integrations.github.repository import GitHubRepositoryProvider
 from sentry.integrations.services.integration import integration_service
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.organizations.services.organization.model import RpcOrganization
+from sentry.plugins.providers.integration_repository import RepositoryConfig
 from sentry.shared_integrations.exceptions import ApiError, IntegrationError
 
 WEBHOOK_EVENTS = ["push", "pull_request"]
@@ -25,7 +28,9 @@ def _validate_repo(self, client, installation, repo):
 
         return repo_data
 
-    def build_repository_config(self, organization: RpcOrganization, data):
+    def build_repository_config(
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
         integration = integration_service.get_integration(
             integration_id=data["integration_id"], provider=self.repo_provider
         )

src/sentry/integrations/gitlab/repository.py

@@ -1,6 +1,9 @@
+from typing import Any
+
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.organizations.services.organization.model import RpcOrganization
 from sentry.plugins.providers import IntegrationRepositoryProvider
+from sentry.plugins.providers.integration_repository import RepositoryConfig
 from sentry.shared_integrations.exceptions import ApiError
 
 
@@ -31,7 +34,9 @@ def get_repository_data(self, organization, config):
         )
         return config
 
-    def build_repository_config(self, organization: RpcOrganization, data):
+    def build_repository_config(
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
 
         installation = self.get_installation(data.get("installation"), organization.id)
         client = installation.get_client()

src/sentry/integrations/services/repository/impl.py

@@ -106,7 +106,6 @@ def update_repositories(self, *, organization_id: int, updates: list[RpcReposito
         fields_to_update = set(list(update_mapping.values())[0].keys())
 
         with transaction.atomic(router.db_for_write(Repository)):
-
             repositories = Repository.objects.filter(
                 organization_id=organization_id, id__in=update_mapping.keys()
             )

src/sentry/integrations/vsts/repository.py

@@ -9,6 +9,7 @@
 from sentry.models.repository import Repository
 from sentry.organizations.services.organization.model import RpcOrganization
 from sentry.plugins.providers import IntegrationRepositoryProvider
+from sentry.plugins.providers.integration_repository import RepositoryConfig
 
 MAX_COMMIT_DATA_REQUESTS = 90
 
@@ -46,8 +47,8 @@ def get_repository_data(
         return config
 
     def build_repository_config(
-        self, organization: RpcOrganization, data: Mapping[str, str]
-    ) -> Mapping[str, Any]:
+        self, organization: RpcOrganization, data: dict[str, Any]
+    ) -> RepositoryConfig:
         return {
             "name": data["name"],
             "external_id": data["external_id"],