diff --git a/src/sentry/integrations/source_code_management/repo_trees.py b/src/sentry/integrations/source_code_management/repo_trees.py index 693516ad98311b..67514f2e081cc5 100644 --- a/src/sentry/integrations/source_code_management/repo_trees.py +++ b/src/sentry/integrations/source_code_management/repo_trees.py @@ -6,7 +6,7 @@ from typing import Any, NamedTuple from sentry.integrations.services.integration import RpcOrganizationIntegration -from sentry.issues.auto_source_code_config.utils import get_supported_extensions +from sentry.issues.auto_source_code_config.utils.platform import get_supported_extensions from sentry.shared_integrations.exceptions import ApiError, IntegrationError from sentry.utils import metrics from sentry.utils.cache import cache diff --git a/src/sentry/issues/auto_source_code_config/code_mapping.py b/src/sentry/issues/auto_source_code_config/code_mapping.py index 450c0a7e641d7b..3502142ece3da6 100644 --- a/src/sentry/issues/auto_source_code_config/code_mapping.py +++ b/src/sentry/issues/auto_source_code_config/code_mapping.py @@ -20,7 +20,7 @@ from .constants import METRIC_PREFIX from .integration_utils import InstallationNotFoundError, get_installation -from .utils import PlatformConfig +from .utils.platform import PlatformConfig logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def derive_code_mappings( trees_helper = CodeMappingTreesHelper(trees) try: frame_filename = FrameInfo(frame, platform) - return trees_helper.list_file_matches(frame_filename) + return trees_helper.get_file_and_repo_matches(frame_filename) except NeedsExtension: logger.warning("Needs extension: %s", frame.get("filename")) @@ -190,7 +190,7 @@ def generate_code_mappings( return list(self.code_mappings.values()) - def list_file_matches(self, frame_filename: FrameInfo) -> list[dict[str, str]]: + def get_file_and_repo_matches(self, frame_filename: FrameInfo) -> list[dict[str, str]]: """List all the files in a repo that match the frame_filename""" file_matches = [] for repo_full_name in self.trees.keys(): @@ -425,11 +425,8 @@ def convert_stacktrace_frame_path_to_source_path( def create_code_mapping( organization: Organization, + code_mapping: CodeMapping, project: Project, - stacktrace_root: str, - source_path: str, - repo_name: str, - branch: str, ) -> RepositoryProjectPathConfig: installation = get_installation(organization) # It helps with typing since org_integration can be None @@ -437,21 +434,22 @@ def create_code_mapping( raise InstallationNotFoundError repository, _ = Repository.objects.get_or_create( - name=repo_name, + name=code_mapping.repo.name, organization_id=organization.id, defaults={"integration_id": installation.model.id}, ) new_code_mapping, _ = RepositoryProjectPathConfig.objects.update_or_create( project=project, - stack_root=stacktrace_root, + stack_root=code_mapping.stacktrace_root, defaults={ "repository": repository, "organization_id": organization.id, "integration_id": installation.model.id, "organization_integration_id": installation.org_integration.id, - "source_root": source_path, - "default_branch": branch, - "automatically_generated": True, + "source_root": code_mapping.source_path, + "default_branch": code_mapping.repo.branch, + # This function is called from the UI, thus, we know that the code mapping is user generated + "automatically_generated": False, }, ) diff --git a/src/sentry/issues/auto_source_code_config/derived_code_mappings_endpoint.py b/src/sentry/issues/auto_source_code_config/derived_code_mappings_endpoint.py new file mode 100644 index 00000000000000..911c490f4abcfd --- /dev/null +++ b/src/sentry/issues/auto_source_code_config/derived_code_mappings_endpoint.py @@ -0,0 +1,45 @@ +import logging + +from rest_framework.request import Request + +from sentry.integrations.source_code_management.repo_trees import ( + RepoAndBranch, + RepoTreesIntegration, +) +from sentry.models.organization import Organization + +from .code_mapping import CodeMapping, CodeMappingTreesHelper, FrameInfo +from .integration_utils import get_installation + +logger = logging.getLogger(__name__) + + +def get_file_and_repo_matches(request: Request, organization: Organization) -> list[dict[str, str]]: + frame_info = get_frame_info_from_request(request) + installation = get_installation(organization) + if not isinstance(installation, RepoTreesIntegration): + return [] + trees = installation.get_trees_for_org() + trees_helper = CodeMappingTreesHelper(trees) + return trees_helper.get_file_and_repo_matches(frame_info) + + +def get_frame_info_from_request(request: Request) -> FrameInfo: + frame = { + "absPath": request.GET.get("absPath"), + # Currently, the only required parameter, thus, avoiding the `get` method + "filename": request.GET["stacktraceFilename"], + "module": request.GET.get("module"), + } + return FrameInfo(frame, request.GET.get("platform")) + + +def get_code_mapping_from_request(request: Request) -> CodeMapping: + return CodeMapping( + repo=RepoAndBranch( + name=request.data["repoName"], + branch=request.data["defaultBranch"], + ), + stacktrace_root=request.data["stackRoot"], + source_path=request.data["sourceRoot"], + ) diff --git a/src/sentry/issues/auto_source_code_config/in_app_stack_trace_rules.py b/src/sentry/issues/auto_source_code_config/in_app_stack_trace_rules.py index b3e46746bf65ad..76f7528bde0ce9 100644 --- a/src/sentry/issues/auto_source_code_config/in_app_stack_trace_rules.py +++ b/src/sentry/issues/auto_source_code_config/in_app_stack_trace_rules.py @@ -2,7 +2,7 @@ from collections.abc import Sequence from sentry.issues.auto_source_code_config.code_mapping import CodeMapping -from sentry.issues.auto_source_code_config.utils import PlatformConfig +from sentry.issues.auto_source_code_config.utils.platform import PlatformConfig from sentry.models.project import Project from sentry.utils import metrics diff --git a/src/sentry/issues/auto_source_code_config/stacktraces.py b/src/sentry/issues/auto_source_code_config/stacktraces.py index aba8770757262c..b8314f7ea59438 100644 --- a/src/sentry/issues/auto_source_code_config/stacktraces.py +++ b/src/sentry/issues/auto_source_code_config/stacktraces.py @@ -6,7 +6,7 @@ from sentry.db.models.fields.node import NodeData from sentry.utils.safe import get_path -from .utils import PlatformConfig +from .utils.platform import PlatformConfig logger = logging.getLogger(__name__) diff --git a/src/sentry/issues/auto_source_code_config/task.py b/src/sentry/issues/auto_source_code_config/task.py index 89f7492141c4d8..6a8afaefec2d32 100644 --- a/src/sentry/issues/auto_source_code_config/task.py +++ b/src/sentry/issues/auto_source_code_config/task.py @@ -32,7 +32,8 @@ get_installation, ) from .stacktraces import get_frames_to_process -from .utils import PlatformConfig +from .utils.platform import PlatformConfig +from .utils.repository import create_repository logger = logging.getLogger(__name__) @@ -202,29 +203,6 @@ def create_configurations( return code_mappings, in_app_stack_trace_rules -def create_repository( - repo_name: str, org_integration: RpcOrganizationIntegration, tags: Mapping[str, str | bool] -) -> Repository | None: - organization_id = org_integration.organization_id - created = False - repository = ( - Repository.objects.filter(name=repo_name, organization_id=organization_id) - .order_by("-date_added") - .first() - ) - if not repository: - if not tags["dry_run"]: - repository, created = Repository.objects.get_or_create( - name=repo_name, - organization_id=organization_id, - integration_id=org_integration.integration_id, - ) - if created or tags["dry_run"]: - metrics.incr(key=f"{METRIC_PREFIX}.repository.created", tags=tags, sample_rate=1.0) - - return repository - - def create_code_mapping( code_mapping: CodeMapping, repository: Repository | None, diff --git a/src/sentry/issues/auto_source_code_config/utils/__init__.py b/src/sentry/issues/auto_source_code_config/utils/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/issues/auto_source_code_config/utils.py b/src/sentry/issues/auto_source_code_config/utils/platform.py similarity index 97% rename from src/sentry/issues/auto_source_code_config/utils.py rename to src/sentry/issues/auto_source_code_config/utils/platform.py index a298b242d1ea84..31260e719caeb8 100644 --- a/src/sentry/issues/auto_source_code_config/utils.py +++ b/src/sentry/issues/auto_source_code_config/utils/platform.py @@ -3,7 +3,7 @@ from sentry import features from sentry.models.organization import Organization -from .constants import PLATFORMS_CONFIG +from ..constants import PLATFORMS_CONFIG def supported_platform(platform: str) -> bool: diff --git a/src/sentry/issues/auto_source_code_config/utils/repository.py b/src/sentry/issues/auto_source_code_config/utils/repository.py new file mode 100644 index 00000000000000..b3f8a2f514e0b2 --- /dev/null +++ b/src/sentry/issues/auto_source_code_config/utils/repository.py @@ -0,0 +1,30 @@ +from collections.abc import Mapping + +from sentry.integrations.services.integration.model import RpcOrganizationIntegration +from sentry.models.repository import Repository +from sentry.utils import metrics + +from ..constants import METRIC_PREFIX + + +def create_repository( + repo_name: str, org_integration: RpcOrganizationIntegration, tags: Mapping[str, str | bool] +) -> Repository | None: + organization_id = org_integration.organization_id + created = False + repository = ( + Repository.objects.filter(name=repo_name, organization_id=organization_id) + .order_by("-date_added") + .first() + ) + if not repository: + if not tags["dry_run"]: + repository, created = Repository.objects.get_or_create( + name=repo_name, + organization_id=organization_id, + integration_id=org_integration.integration_id, + ) + if created or tags["dry_run"]: + metrics.incr(key=f"{METRIC_PREFIX}.repository.created", tags=tags, sample_rate=1.0) + + return repository diff --git a/src/sentry/issues/endpoints/organization_derive_code_mappings.py b/src/sentry/issues/endpoints/organization_derive_code_mappings.py index de008cbb7e2244..80a99f90480727 100644 --- a/src/sentry/issues/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/issues/endpoints/organization_derive_code_mappings.py @@ -1,3 +1,4 @@ +import logging from typing import Literal from rest_framework import status @@ -12,17 +13,21 @@ OrganizationIntegrationsLoosePermission, ) from sentry.api.serializers import serialize -from sentry.issues.auto_source_code_config.code_mapping import ( - create_code_mapping, - derive_code_mappings, +from sentry.issues.auto_source_code_config.code_mapping import NeedsExtension, create_code_mapping +from sentry.issues.auto_source_code_config.derived_code_mappings_endpoint import ( + get_code_mapping_from_request, + get_file_and_repo_matches, ) from sentry.issues.auto_source_code_config.integration_utils import ( InstallationCannotGetTreesError, InstallationNotFoundError, + get_installation, ) from sentry.models.organization import Organization from sentry.models.project import Project +logger = logging.getLogger(__name__) + @region_silo_endpoint class OrganizationDeriveCodeMappingsEndpoint(OrganizationEndpoint): @@ -33,39 +38,34 @@ class OrganizationDeriveCodeMappingsEndpoint(OrganizationEndpoint): owner = ApiOwner.ISSUES publish_status = { - "GET": ApiPublishStatus.UNKNOWN, - "POST": ApiPublishStatus.UNKNOWN, + "GET": ApiPublishStatus.EXPERIMENTAL, + "POST": ApiPublishStatus.EXPERIMENTAL, } permission_classes = (OrganizationIntegrationsLoosePermission,) def get(self, request: Request, organization: Organization) -> Response: """ - Get all matches for a stacktrace filename. + Get all files from the customer repositories that match a stack trace frame. `````````````````` :param organization: + :param string absPath: + :param string module: :param string stacktraceFilename: :param string platform: :auth: required """ - stacktrace_filename = request.GET.get("stacktraceFilename") - # XXX: The UI will need to pass the platform - platform = request.GET.get("platform") - try: - possible_code_mappings = [] + file_repo_matches = [] resp_status: Literal[200, 204, 400] = status.HTTP_400_BAD_REQUEST - if stacktrace_filename: - possible_code_mappings = derive_code_mappings( - organization, {"filename": stacktrace_filename}, platform - ) - if possible_code_mappings: - resp_status = status.HTTP_200_OK - else: - resp_status = status.HTTP_204_NO_CONTENT + file_repo_matches = get_file_and_repo_matches(request, organization) + if file_repo_matches: + resp_status = status.HTTP_200_OK + else: + resp_status = status.HTTP_204_NO_CONTENT - return Response(serialize(possible_code_mappings), status=resp_status) + return self.respond(serialize(file_repo_matches), status=resp_status) except InstallationCannotGetTreesError: return self.respond( {"text": "The integration does not support getting trees"}, @@ -76,6 +76,12 @@ def get(self, request: Request, organization: Organization) -> Response: {"text": "Could not find this integration installed on your organization"}, status=status.HTTP_404_NOT_FOUND, ) + except NeedsExtension: + return self.respond({"text": "Needs extension"}, status=status.HTTP_400_BAD_REQUEST) + except KeyError: + return self.respond( + {"text": "Missing required parameters"}, status=status.HTTP_400_BAD_REQUEST + ) def post(self, request: Request, organization: Organization) -> Response: """ @@ -100,19 +106,18 @@ def post(self, request: Request, organization: Organization) -> Response: if not request.access.has_project_access(project): return self.respond(status=status.HTTP_403_FORBIDDEN) - repo_name = request.data.get("repoName") - stack_root = request.data.get("stackRoot") - source_root = request.data.get("sourceRoot") - branch = request.data.get("defaultBranch") - if not repo_name or not stack_root or not source_root or not branch: + try: + installation = get_installation(organization) + # It helps with typing since org_integration can be None + if not installation.org_integration: + raise InstallationNotFoundError + + code_mapping = get_code_mapping_from_request(request) + new_code_mapping = create_code_mapping(organization, code_mapping, project) + except KeyError: return self.respond( {"text": "Missing required parameters"}, status=status.HTTP_400_BAD_REQUEST ) - - try: - new_code_mapping = create_code_mapping( - organization, project, stack_root, source_root, repo_name, branch - ) except InstallationNotFoundError: return self.respond( {"text": "Could not find this integration installed on your organization"}, diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 855edc363b8bfa..59493e52eb6c6e 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -1007,7 +1007,7 @@ def process_code_mappings(job: PostProcessJob) -> None: return from sentry.issues.auto_source_code_config.stacktraces import get_frames_to_process - from sentry.issues.auto_source_code_config.utils import supported_platform + from sentry.issues.auto_source_code_config.utils.platform import supported_platform from sentry.tasks.auto_source_code_config import auto_source_code_config try: diff --git a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py index 099a6b777e541f..7b2ebe79b86ad5 100644 --- a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py +++ b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py @@ -51,7 +51,7 @@ def test_get_single_match(self, mock_get_trees_for_org: Any) -> None: } ] with patch( - "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.get_file_and_repo_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -73,7 +73,7 @@ def test_get_start_with_backslash(self, mock_get_trees_for_org: Any) -> None: } ] with patch( - "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.get_file_and_repo_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -103,7 +103,7 @@ def test_get_multiple_matches(self, mock_get_trees_for_org: Any) -> None: }, ] with patch( - "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.get_file_and_repo_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -156,7 +156,7 @@ def test_post_simple(self) -> None: repo = Repository.objects.get(name="getsentry/codemap") assert response.status_code == 201, response.content assert response.data == { - "automaticallyGenerated": True, + "automaticallyGenerated": False, "id": str(response.data["id"]), "projectId": str(self.project.id), "projectSlug": self.project.slug, diff --git a/tests/sentry/issues/auto_source_code_config/test_code_mapping.py b/tests/sentry/issues/auto_source_code_config/test_code_mapping.py index 42fc4ad70e3ad1..12bd75c9b33275 100644 --- a/tests/sentry/issues/auto_source_code_config/test_code_mapping.py +++ b/tests/sentry/issues/auto_source_code_config/test_code_mapping.py @@ -320,9 +320,9 @@ def test_more_than_one_repo_match(self, logger: Any) -> None: assert code_mappings == [] logger.warning.assert_called_with("More than one repo matched %s", "sentry/web/urls.py") - def test_list_file_matches_single(self) -> None: + def test_get_file_and_repo_matches_single(self) -> None: frame_filename = FrameInfo({"filename": "sentry_plugins/slack/client.py"}) - matches = self.code_mapping_helper.list_file_matches(frame_filename) + matches = self.code_mapping_helper.get_file_and_repo_matches(frame_filename) expected_matches = [ { "filename": "src/sentry_plugins/slack/client.py", @@ -334,9 +334,9 @@ def test_list_file_matches_single(self) -> None: ] assert matches == expected_matches - def test_list_file_matches_multiple(self) -> None: + def test_get_file_and_repo_matches_multiple(self) -> None: frame_filename = FrameInfo({"filename": "sentry/web/urls.py"}) - matches = self.code_mapping_helper.list_file_matches(frame_filename) + matches = self.code_mapping_helper.get_file_and_repo_matches(frame_filename) expected_matches = [ { "filename": "src/sentry/web/urls.py", diff --git a/tests/sentry/issues/auto_source_code_config/test_process_event.py b/tests/sentry/issues/auto_source_code_config/test_process_event.py index 196f6993ce5524..d55c2b88ec6804 100644 --- a/tests/sentry/issues/auto_source_code_config/test_process_event.py +++ b/tests/sentry/issues/auto_source_code_config/test_process_event.py @@ -11,7 +11,7 @@ ) from sentry.issues.auto_source_code_config.integration_utils import InstallationNotFoundError from sentry.issues.auto_source_code_config.task import DeriveCodeMappingsErrorReason, process_event -from sentry.issues.auto_source_code_config.utils import PlatformConfig +from sentry.issues.auto_source_code_config.utils.platform import PlatformConfig from sentry.models.repository import Repository from sentry.shared_integrations.exceptions import ApiError from sentry.testutils.asserts import assert_failure_metric, assert_halt_metric @@ -246,7 +246,7 @@ class TestGenericBehaviour(BaseDeriveCodeMappings): """Behaviour that is not specific to a language.""" def test_skips_not_supported_platforms(self) -> None: - with patch(f"{CODE_ROOT}.utils.get_platform_config", return_value={}): + with patch(f"{CODE_ROOT}.utils.platform.get_platform_config", return_value={}): self._process_and_assert_configuration_changes( repo_trees={}, frames=[{}], platform="other" ) @@ -284,9 +284,11 @@ def test_dry_run_platform(self) -> None: file_in_repo = "src/foo/bar.py" platform = "other" with ( - patch(f"{CODE_ROOT}.utils.get_platform_config", return_value={}), - patch(f"{CODE_ROOT}.utils.PlatformConfig.is_supported", return_value=True), - patch(f"{CODE_ROOT}.utils.PlatformConfig.is_dry_run_platform", return_value=True), + patch(f"{CODE_ROOT}.utils.platform.get_platform_config", return_value={}), + patch(f"{CODE_ROOT}.utils.platform.PlatformConfig.is_supported", return_value=True), + patch( + f"{CODE_ROOT}.utils.platform.PlatformConfig.is_dry_run_platform", return_value=True + ), ): # No code mapping will be stored, however, we get what would have been created self._process_and_assert_configuration_changes( @@ -303,7 +305,7 @@ def test_extension_is_not_included(self) -> None: self.event = self.create_event([{"filename": frame_filename, "in_app": True}], platform) with ( - patch(f"{CODE_ROOT}.utils.get_platform_config", return_value={}), + patch(f"{CODE_ROOT}.utils.platform.get_platform_config", return_value={}), patch(f"{REPO_TREES_CODE}.get_supported_extensions", return_value=[]), ): # No extensions are supported, thus, we won't generate a code mapping @@ -329,8 +331,8 @@ def test_multiple_calls(self) -> None: REPO2: ["app/baz/qux.py"], } with ( - patch(f"{CODE_ROOT}.utils.get_platform_config", return_value={}), - patch(f"{CODE_ROOT}.utils.PlatformConfig.is_supported", return_value=True), + patch(f"{CODE_ROOT}.utils.platform.get_platform_config", return_value={}), + patch(f"{CODE_ROOT}.utils.platform.PlatformConfig.is_supported", return_value=True), ): self._process_and_assert_configuration_changes( repo_trees=repo_trees, diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index b6981e8fe1ddac..3ddaad840d7c60 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -21,7 +21,7 @@ from sentry.feedback.usecases.create_feedback import FeedbackCreationSource from sentry.integrations.models.integration import Integration from sentry.integrations.source_code_management.commit_context import CommitInfo, FileBlameInfo -from sentry.issues.auto_source_code_config.utils import get_supported_platforms +from sentry.issues.auto_source_code_config.utils.platform import get_supported_platforms from sentry.issues.grouptype import ( FeedbackGroup, GroupCategory,