diff --git a/src/sentry/sentry_metrics/querying/data/api.py b/src/sentry/sentry_metrics/querying/data/api.py index 5cc57e34de4448..a9cefc4c0fff2e 100644 --- a/src/sentry/sentry_metrics/querying/data/api.py +++ b/src/sentry/sentry_metrics/querying/data/api.py @@ -1,6 +1,5 @@ from collections.abc import Sequence from datetime import datetime -from typing import cast from snuba_sdk import MetricsQuery, MetricsScope, Rollup @@ -8,12 +7,23 @@ from sentry.models.environment import Environment from sentry.models.organization import Organization from sentry.models.project import Project -from sentry.sentry_metrics.querying.data.execution import QueryExecutor, QueryResult +from sentry.sentry_metrics.querying.data.execution import QueryExecutor +from sentry.sentry_metrics.querying.data.modulation.modulation_value_map import ( + QueryModulationValueMap, +) +from sentry.sentry_metrics.querying.data.modulation.modulator import ( + Modulator, + Project2ProjectIDModulator, +) from sentry.sentry_metrics.querying.data.parsing import QueryParser +from sentry.sentry_metrics.querying.data.postprocessing.base import run_postprocessing_steps +from sentry.sentry_metrics.querying.data.postprocessing.demodulation import QueryDemodulationStep from sentry.sentry_metrics.querying.data.preparation.base import ( IntermediateQuery, + PreparationStep, run_preparation_steps, ) +from sentry.sentry_metrics.querying.data.preparation.modulation import QueryModulationStep from sentry.sentry_metrics.querying.data.preparation.units_normalization import ( UnitsNormalizationStep, ) @@ -62,12 +72,17 @@ def run_queries( ) ) - preparation_steps = [] + preparation_steps: list[PreparationStep] = [] + modulation_value_map = QueryModulationValueMap() + modulators: list[Modulator] = [Project2ProjectIDModulator()] + if features.has( "organizations:ddm-metrics-api-unit-normalization", organization=organization, actor=None ): preparation_steps.append(UnitsNormalizationStep()) + preparation_steps.append(QueryModulationStep(projects, modulators, modulation_value_map)) + # We run a series of preparation steps which operate on the entire list of queries. intermediate_queries = run_preparation_steps(intermediate_queries, *preparation_steps) @@ -77,6 +92,9 @@ def run_queries( executor.schedule(intermediate_query=intermediate_query, query_type=query_type) results = executor.execute() + results = run_postprocessing_steps( + results, QueryDemodulationStep(projects, modulators, modulation_value_map) + ) # We wrap the result in a class that exposes some utils methods to operate on results. - return MQLQueriesResult(cast(list[QueryResult], results)) + return MQLQueriesResult(results) diff --git a/src/sentry/sentry_metrics/querying/data/execution.py b/src/sentry/sentry_metrics/querying/data/execution.py index 28a74679d8a4db..dc012b8b6f65b3 100644 --- a/src/sentry/sentry_metrics/querying/data/execution.py +++ b/src/sentry/sentry_metrics/querying/data/execution.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from datetime import datetime from enum import Enum from typing import Any, Union, cast @@ -11,6 +11,7 @@ from sentry.models.organization import Organization from sentry.models.project import Project from sentry.sentry_metrics.querying.constants import SNUBA_QUERY_LIMIT +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator from sentry.sentry_metrics.querying.data.preparation.base import IntermediateQuery from sentry.sentry_metrics.querying.data.utils import adjust_time_bounds_with_interval from sentry.sentry_metrics.querying.errors import ( @@ -145,6 +146,7 @@ class ScheduledQuery: unit_family: UnitFamily | None = None unit: MeasurementUnit | None = None scaling_factor: float | None = None + modulators: list[Modulator] = field(default_factory=list) def initialize( self, @@ -318,7 +320,7 @@ def _align_date_range(cls, metrics_query: MetricsQuery) -> tuple[MetricsQuery, i return metrics_query, None -@dataclass(frozen=True) +@dataclass class QueryResult: """ Represents the result of a ScheduledQuery containing its associated series and totals results. @@ -445,12 +447,24 @@ def modified_end(self) -> datetime: @property def series(self) -> Sequence[Mapping[str, Any]]: + if "series" not in self.result: + return [] return self.result["series"]["data"] + @series.setter + def series(self, value: Sequence[Mapping[str, Any]]) -> None: + self.result["series"]["data"] = value + @property def totals(self) -> Sequence[Mapping[str, Any]]: + if "totals" not in self.result: + return [] return self.result["totals"]["data"] + @totals.setter + def totals(self, value: Sequence[Mapping[str, Any]]) -> None: + self.result["totals"]["data"] = value + @property def meta(self) -> Sequence[Mapping[str, str]]: # By default, we extract the metadata from the totals query, if that is not there we extract from the series @@ -464,7 +478,11 @@ def group_bys(self) -> list[str]: # that we can correctly render groups in case they are not returned from the db because of missing data. # # Sorting of the groups is done to maintain consistency across function calls. - return sorted(UsedGroupBysVisitor().visit(self._any_query().metrics_query.query)) + scheduled_query = self._any_query() + modulators = scheduled_query.modulators + return sorted( + UsedGroupBysVisitor(modulators=modulators).visit(scheduled_query.metrics_query.query) + ) @property def interval(self) -> int | None: @@ -774,7 +792,7 @@ def _execution_loop(self): while continue_execution: continue_execution = self._bulk_execute() - def execute(self) -> Sequence[QueryResult]: + def execute(self) -> list[QueryResult]: """ Executes the scheduled queries in the execution loop. @@ -798,7 +816,7 @@ def execute(self) -> Sequence[QueryResult]: "Not all queries were executed in the execution loop" ) - return cast(Sequence[QueryResult], self._query_results) + return cast(list[QueryResult], self._query_results) def schedule(self, intermediate_query: IntermediateQuery, query_type: QueryType): """ @@ -813,6 +831,7 @@ def schedule(self, intermediate_query: IntermediateQuery, query_type: QueryType) unit_family=intermediate_query.unit_family, unit=intermediate_query.unit, scaling_factor=intermediate_query.scaling_factor, + modulators=intermediate_query.modulators, ) # In case the user chooses to run also a series query, we will duplicate the query and chain it after totals. diff --git a/src/sentry/sentry_metrics/querying/data/modulation/__init__.py b/src/sentry/sentry_metrics/querying/data/modulation/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/sentry_metrics/querying/data/modulation/modulation_value_map.py b/src/sentry/sentry_metrics/querying/data/modulation/modulation_value_map.py new file mode 100644 index 00000000000000..cab03e1e4dbaee --- /dev/null +++ b/src/sentry/sentry_metrics/querying/data/modulation/modulation_value_map.py @@ -0,0 +1,13 @@ +class QueryModulationValueMap(dict): + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def __getitem__(self, key): + return dict.__getitem__(self, key) + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + + def update(self, *args, **kwargs): + for key, value in dict(*args, **kwargs).items(): + self[key] = value diff --git a/src/sentry/sentry_metrics/querying/data/modulation/modulator.py b/src/sentry/sentry_metrics/querying/data/modulation/modulator.py new file mode 100644 index 00000000000000..2e6ecab11bf0e1 --- /dev/null +++ b/src/sentry/sentry_metrics/querying/data/modulation/modulator.py @@ -0,0 +1,84 @@ +import abc +from collections.abc import Sequence + +from snuba_sdk import Formula + +from sentry.models.project import Project +from sentry.sentry_metrics.querying.data.modulation.modulation_value_map import ( + QueryModulationValueMap, +) + + +class Modulator(abc.ABC): + def __init__(self, from_key: str, to_key: str): + self.from_key = from_key + self.to_key = to_key + + def __hash__(self): + return hash((self.from_key, self.to_key)) + + @abc.abstractmethod + def modulate( + self, + projects: Sequence[Project], + value_map: QueryModulationValueMap, + formula: Formula, + **kwargs, + ) -> Formula: + return formula + + @abc.abstractmethod + def demodulate( + self, + projects: Sequence[Project], + value_map: QueryModulationValueMap, + formula: Formula, + **kwargs, + ) -> Formula: + return formula + + +class Project2ProjectIDModulator(Modulator): + def __init__(self, from_key: str = "project", to_key: str = "project_id"): + self.from_key = from_key + self.to_key = to_key + + def modulate( + self, + projects: Sequence[Project], + value_map: QueryModulationValueMap, + formula: Formula, + ) -> Formula: + if formula not in value_map: + value_map[formula] = None + for project in projects: + if project.slug == formula: + value_map[formula] = project.id + return value_map[formula] + + def demodulate( + self, + projects: Sequence[Project], + value_map: QueryModulationValueMap, + formula: Formula, + ) -> Formula: + if formula not in value_map: + for project in projects: + if project.id == formula: + value_map[formula] = project.slug + + return value_map[formula] + + +def find_modulator( + modulators: Sequence[Modulator], from_key: str | None = None, to_key: str | None = None +) -> Modulator | None: + for modulator in modulators: + if from_key: + if modulator.from_key == from_key: + return modulator + if to_key: + if modulator.to_key == to_key: + return modulator + + return None diff --git a/src/sentry/sentry_metrics/querying/data/postprocessing/__init__.py b/src/sentry/sentry_metrics/querying/data/postprocessing/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/sentry_metrics/querying/data/postprocessing/base.py b/src/sentry/sentry_metrics/querying/data/postprocessing/base.py new file mode 100644 index 00000000000000..d16f8f056d94e9 --- /dev/null +++ b/src/sentry/sentry_metrics/querying/data/postprocessing/base.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod + +from sentry.sentry_metrics.querying.data.execution import QueryResult + + +class PostProcessingStep(ABC): + """ + Represents an abstract step that post-processes a collection of QueryResult objects. + + The post-processing of these objects might include transforming them or just obtaining some intermediate data that + is useful to compute other things before returning the results. + """ + + @abstractmethod + def run(self, query_results: list[QueryResult]) -> list[QueryResult]: + """ + Runs the post-processing steps on a list of query results. + + Returns: + A list of post-processed query results. + """ + raise NotImplementedError + + +def run_postprocessing_steps(query_results: list[QueryResult], *steps) -> list[QueryResult]: + """ + Takes a series of query results and steps and runs the post-processing steps one after each other in order they are + supplied in. + + Returns: + A list of query results after running the post-processing steps. + """ + for step in steps: + if isinstance(step, PostProcessingStep): + query_results = step.run(query_results=query_results) + + return query_results diff --git a/src/sentry/sentry_metrics/querying/data/postprocessing/demodulation.py b/src/sentry/sentry_metrics/querying/data/postprocessing/demodulation.py new file mode 100644 index 00000000000000..85fb6455f20289 --- /dev/null +++ b/src/sentry/sentry_metrics/querying/data/postprocessing/demodulation.py @@ -0,0 +1,53 @@ +from collections.abc import Mapping, Sequence +from typing import Any + +from sentry.models.project import Project +from sentry.sentry_metrics.querying.data.execution import QueryResult +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator +from sentry.sentry_metrics.querying.data.postprocessing.base import PostProcessingStep + + +class QueryDemodulationStep(PostProcessingStep): + def __init__( + self, + projects: Sequence[Project], + modulators: Sequence[Modulator], + value_map: dict[Any, Any], + ): + self.projects = projects + self.modulators = modulators + self.value_map = value_map + + def run(self, query_results: list[QueryResult]) -> list[QueryResult]: + for query_result in query_results: + if query_result.totals: + query_result.totals = self._demodulate_data( + query_result.totals, query_result.totals_query.modulators + ) + if query_result.series: + query_result.series = self._demodulate_data( + query_result.series, query_result.series_query.modulators + ) + + return query_results + + def _demodulate_data( + self, data: Sequence[Mapping[str, Any]], modulators: list[Modulator] + ) -> Sequence[Mapping[str, Any]]: + for element in data: + updated_element = dict() + keys_to_delete = [] + for result_key in element.keys(): + for modulator in modulators: + if modulator.to_key == result_key: + original_value = modulator.demodulate( + self.projects, self.value_map, element[result_key] + ) + updated_element[modulator.from_key] = original_value + keys_to_delete.append(result_key) + + for key in keys_to_delete: + del element[key] + element.update(updated_element) + + return data diff --git a/src/sentry/sentry_metrics/querying/data/preparation/base.py b/src/sentry/sentry_metrics/querying/data/preparation/base.py index 51874593102460..39a2519bab571c 100644 --- a/src/sentry/sentry_metrics/querying/data/preparation/base.py +++ b/src/sentry/sentry_metrics/querying/data/preparation/base.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from snuba_sdk import MetricsQuery +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator from sentry.sentry_metrics.querying.types import QueryOrder from sentry.sentry_metrics.querying.units import MeasurementUnit, UnitFamily @@ -27,6 +28,7 @@ class IntermediateQuery: unit_family: UnitFamily | None = None unit: MeasurementUnit | None = None scaling_factor: float | None = None + modulators: list[Modulator] = field(default_factory=list) class PreparationStep(ABC): diff --git a/src/sentry/sentry_metrics/querying/data/preparation/modulation.py b/src/sentry/sentry_metrics/querying/data/preparation/modulation.py new file mode 100644 index 00000000000000..a0b005b2d8a8ba --- /dev/null +++ b/src/sentry/sentry_metrics/querying/data/preparation/modulation.py @@ -0,0 +1,44 @@ +from collections.abc import Sequence +from dataclasses import replace + +from sentry.models.project import Project +from sentry.sentry_metrics.querying.data.modulation.modulation_value_map import ( + QueryModulationValueMap, +) +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator +from sentry.sentry_metrics.querying.data.preparation.base import IntermediateQuery, PreparationStep +from sentry.sentry_metrics.querying.visitors.query_expression import ModulatorVisitor + + +class QueryModulationStep(PreparationStep): + def __init__( + self, + projects: Sequence[Project], + modulators: list[Modulator], + value_map: QueryModulationValueMap, + ): + self.projects = projects + self.modulators = modulators + self.value_map = value_map + + def _get_modulated_intermediate_query( + self, intermediate_query: IntermediateQuery + ) -> IntermediateQuery: + visitor = ModulatorVisitor(self.projects, self.modulators, self.value_map) + modulated_query = visitor.visit(intermediate_query.metrics_query.query) + + return replace( + intermediate_query, + metrics_query=intermediate_query.metrics_query.set_query(modulated_query), + modulators=visitor.applied_modulators, + ) + + def run(self, intermediate_queries: list[IntermediateQuery]) -> list[IntermediateQuery]: + modulated_intermediate_queries = [] + + for intermediate_query in intermediate_queries: + modulated_intermediate_queries.append( + self._get_modulated_intermediate_query(intermediate_query) + ) + + return modulated_intermediate_queries diff --git a/src/sentry/sentry_metrics/querying/visitors/query_condition.py b/src/sentry/sentry_metrics/querying/visitors/query_condition.py index 78683adce29e0f..4c5852f3d9179d 100644 --- a/src/sentry/sentry_metrics/querying/visitors/query_condition.py +++ b/src/sentry/sentry_metrics/querying/visitors/query_condition.py @@ -4,9 +4,13 @@ from sentry.api.serializers import bulk_fetch_project_latest_releases from sentry.models.project import Project +from sentry.sentry_metrics.querying.data.modulation.modulation_value_map import ( + QueryModulationValueMap, +) +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator, find_modulator from sentry.sentry_metrics.querying.errors import LatestReleaseNotFoundError from sentry.sentry_metrics.querying.types import QueryCondition -from sentry.sentry_metrics.querying.visitors.base import QueryConditionVisitor +from sentry.sentry_metrics.querying.visitors.base import QueryConditionVisitor, TVisited class LatestReleaseTransformationVisitor(QueryConditionVisitor[QueryCondition]): @@ -96,3 +100,44 @@ def _visit_condition(self, condition: Condition) -> QueryCondition: op=condition.op, rhs=condition.rhs, ) + + +class ModulatorConditionVisitor(QueryConditionVisitor): + def __init__( + self, + projects: Sequence[Project], + modulators: Sequence[Modulator], + value_map: QueryModulationValueMap, + ): + self.projects = projects + self.modulators = modulators + self.applied_modulators = [] + self.value_map = value_map + + def _visit_condition(self, condition: Condition) -> TVisited: + lhs = condition.lhs + rhs = condition.rhs + + if isinstance(lhs, Column): + modulator = find_modulator(self.modulators, lhs.name) + if modulator: + new_lhs = Column(modulator.to_key) + self.applied_modulators.append(modulator) + + if isinstance(rhs, list): + new_rhs = [ + modulator.modulate(self.projects, self.value_map, element) + for element in rhs + ] + else: + new_rhs = modulator.modulate(self.projects, self.value_map, rhs) + return Condition(lhs=new_lhs, op=condition.op, rhs=new_rhs) + + return condition + + def _visit_boolean_condition(self, boolean_condition: BooleanCondition) -> TVisited: + conditions = [] + for condition in boolean_condition.conditions: + conditions.append(self.visit(condition)) + + return BooleanCondition(op=boolean_condition.op, conditions=conditions) diff --git a/src/sentry/sentry_metrics/querying/visitors/query_expression.py b/src/sentry/sentry_metrics/querying/visitors/query_expression.py index 6a7dc79b652801..ed05ec28eb9e39 100644 --- a/src/sentry/sentry_metrics/querying/visitors/query_expression.py +++ b/src/sentry/sentry_metrics/querying/visitors/query_expression.py @@ -4,7 +4,12 @@ from snuba_sdk.conditions import ConditionGroup from sentry.models.environment import Environment +from sentry.models.project import Project from sentry.sentry_metrics.querying.constants import COEFFICIENT_OPERATORS +from sentry.sentry_metrics.querying.data.modulation.modulation_value_map import ( + QueryModulationValueMap, +) +from sentry.sentry_metrics.querying.data.modulation.modulator import Modulator, find_modulator from sentry.sentry_metrics.querying.errors import InvalidMetricsQueryError from sentry.sentry_metrics.querying.types import QueryExpression from sentry.sentry_metrics.querying.units import ( @@ -20,6 +25,7 @@ QueryConditionVisitor, QueryExpressionVisitor, ) +from sentry.sentry_metrics.querying.visitors.query_condition import ModulatorConditionVisitor from sentry.snuba.metrics import parse_mri @@ -245,6 +251,9 @@ class UsedGroupBysVisitor(QueryExpressionVisitor[set[str]]): Visitor that recursively computes all the groups of the `QueryExpression`. """ + def __init__(self, modulators: list[Modulator] = None): + self.modulators = modulators or [] + def _visit_formula(self, formula: Formula) -> set[str]: group_bys: set[str] = set() @@ -271,10 +280,20 @@ def _group_bys_as_string(self, group_bys: list[Column | AliasedExpression] | Non string_group_bys = set() for group_by in group_bys: + modulator = None if isinstance(group_by, AliasedExpression): - string_group_bys.add(group_by.exp.name) + modulator = find_modulator(modulators=self.modulators, to_key=group_by.exp.name) elif isinstance(group_by, Column): - string_group_bys.add(group_by.name) + modulator = find_modulator(modulators=self.modulators, to_key=group_by.name) + + if modulator: + string_group_bys.add(modulator.from_key) + + else: + if isinstance(group_by, AliasedExpression): + string_group_bys.add(group_by.exp.name) + elif isinstance(group_by, Column): + string_group_bys.add(group_by.name) return string_group_bys @@ -437,3 +456,68 @@ def _visit_float(self, float_number: float) -> QueryExpression: def _is_numeric_scalar(self, value: QueryExpression) -> bool: return isinstance(value, int) or isinstance(value, float) + + +class ModulatorVisitor(QueryExpressionVisitor): + """ + Visitor that recursively transforms the QueryExpression components to modulate certain attributes to be queried + by API that need to be translated for Snuba to be able to query the data. + """ + + def __init__( + self, + projects: Sequence[Project], + modulators: Sequence[Modulator], + value_map: QueryModulationValueMap, + ): + self.projects = projects + self.modulators = modulators + self.value_map = value_map + self.applied_modulators: list[Modulator] = [] + + def _visit_formula(self, formula: Formula) -> Formula: + formula = super()._visit_formula(formula) + + filters = ModulatorConditionVisitor( + self.projects, self.modulators, self.value_map + ).visit_group(formula.filters) + formula = formula.set_filters(filters) + + if formula.groupby: + new_group_bys = self._modulate_groupby(formula.groupby) + formula = formula.set_groupby(new_group_bys) + + return formula + + def _visit_timeseries(self, timeseries: Timeseries) -> Timeseries: + filters = ModulatorConditionVisitor( + self.projects, self.modulators, self.value_map + ).visit_group(timeseries.filters) + timeseries = timeseries.set_filters(filters) + + if timeseries.groupby: + new_group_bys = self._modulate_groupby(timeseries.groupby) + timeseries = timeseries.set_groupby(new_group_bys) + + return timeseries + + def _modulate_groupby( + self, groupby: list[Column | AliasedExpression] | None = None + ) -> list[Column | AliasedExpression]: + new_group_bys = [] + for group in groupby: + new_group = group + if isinstance(group, Column): + modulator = find_modulator(self.modulators, group.name) + if modulator: + new_group = Column(name=modulator.to_key) + self.applied_modulators.append(modulator) + elif isinstance(group, AliasedExpression): + modulator = find_modulator(self.modulators, group.exp.name) + if modulator: + new_group = AliasedExpression( + exp=Column(name=modulator.to_key), alias=group.alias + ) + self.applied_modulators.append(modulator) + new_group_bys.append(new_group) + return new_group_bys diff --git a/tests/sentry/sentry_metrics/querying/data/test_api.py b/tests/sentry/sentry_metrics/querying/data/test_api.py index eb70e07d6587cd..345918c32dd996 100644 --- a/tests/sentry/sentry_metrics/querying/data/test_api.py +++ b/tests/sentry/sentry_metrics/querying/data/test_api.py @@ -1649,3 +1649,274 @@ def test_query_with_basic_formula_and_coefficient_operators(self): assert meta[0][1]["unit_family"] == expected_unit_family assert meta[0][1]["unit"] == expected_unit assert meta[0][1]["scaling_factor"] is None + + @with_feature("organizations:ddm-metrics-api-unit-normalization") + def test_condition_using_project_name_gets_visited_correctly(self) -> None: + mql = self.mql("sum", TransactionMRI.DURATION.value, "project:bar") + query = MQLQuery(mql) + + results = self.run_query( + mql_queries=[query], + start=self.now() - timedelta(minutes=30), + end=self.now() + timedelta(hours=1, minutes=30), + interval=3600, + organization=self.project.organization, + projects=[self.project], + environments=[], + referrer="metrics.data.api", + ) + data = results["data"] + assert len(data) == 1 + assert data[0][0]["by"] == {} + assert data[0][0]["series"] == [ + None, + self.to_reference_unit(12.0), + self.to_reference_unit(9.0), + ] + assert data[0][0]["totals"] == self.to_reference_unit(21.0) + + @with_feature("organizations:ddm-metrics-api-unit-normalization") + def test_groupby_using_project_name_gets_visited_correctly(self) -> None: + self.new_project = self.create_project(name="Bar Again") + for value, transaction, platform, env, time in ( + (1, "/hello", "android", "prod", self.now()), + (3, "/hello", "android", "prod", self.now()), + (5, "/hello", "android", "prod", self.now()), + (2, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (5, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (8, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + ): + self.store_metric( + self.new_project.organization.id, + self.new_project.id, + "distribution", + TransactionMRI.DURATION.value, + { + "transaction": transaction, + "platform": platform, + "environment": env, + }, + self.ts(time), + value, + UseCaseID.TRANSACTIONS, + ) + + mql = self.mql("avg", TransactionMRI.DURATION.value, group_by="project") + query = MQLQuery(mql) + + results = self.run_query( + mql_queries=[query], + start=self.now() - timedelta(minutes=30), + end=self.now() + timedelta(hours=1, minutes=30), + interval=3600, + organization=self.project.organization, + projects=[self.project, self.new_project], + environments=[], + referrer="metrics.data.api", + ) + data = results["data"][0] + data = sorted(data, key=lambda x: x["by"]["project"]) + assert len(data) == 2 + assert data[1]["by"] == {"project": self.new_project.slug} + assert data[1]["series"] == [ + None, + self.to_reference_unit(3.0), + self.to_reference_unit(5.0), + ] + assert data[1]["totals"] == self.to_reference_unit(4.0) + + @with_feature("organizations:ddm-metrics-api-unit-normalization") + def test_groupby_and_filter_by_project_name(self) -> None: + self.new_project_1 = self.create_project(name="Bar Again") + self.new_project_2 = self.create_project(name="Bar Yet Again") + for value, transaction, platform, env, time in ( + (1, "/hello", "android", "prod", self.now()), + (3, "/hello", "android", "prod", self.now()), + (5, "/hello", "android", "prod", self.now()), + (2, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (5, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (8, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + ): + self.store_metric( + self.new_project_1.organization.id, + self.new_project_1.id, + "distribution", + TransactionMRI.DURATION.value, + { + "transaction": transaction, + "platform": platform, + "environment": env, + }, + self.ts(time), + value, + UseCaseID.TRANSACTIONS, + ) + + for value, transaction, platform, env, time in ( + (1, "/hello", "android", "prod", self.now()), + (3, "/hello", "android", "prod", self.now()), + (5, "/hello", "android", "prod", self.now()), + (2, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (5, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (8, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + ): + self.store_metric( + self.new_project_2.organization.id, + self.new_project_2.id, + "distribution", + TransactionMRI.DURATION.value, + { + "transaction": transaction, + "platform": platform, + "environment": env, + }, + self.ts(time), + value, + UseCaseID.TRANSACTIONS, + ) + + mqls = [ + self.mql( + "avg", + TransactionMRI.DURATION.value, + group_by="project", + filters="project:[bar,bar-again]", + ), + self.mql( + "avg", + TransactionMRI.DURATION.value, + group_by="project", + filters="!project:bar-yet-again", + ), + self.mql( + "avg", + TransactionMRI.DURATION.value, + group_by="project", + filters="project:bar or project:bar-again", + ), + ] + for mql in mqls: + query = MQLQuery(mql) + + results = self.run_query( + mql_queries=[query], + start=self.now() - timedelta(minutes=30), + end=self.now() + timedelta(hours=1, minutes=30), + interval=3600, + organization=self.project.organization, + projects=[self.project, self.new_project_1, self.new_project_2], + environments=[], + referrer="metrics.data.api", + ) + data = results["data"][0] + assert len(data) == 2 + data = sorted(data, key=lambda x: x["by"]["project"]) + assert data[1]["by"] == {"project": self.new_project_1.slug} + assert data[1]["series"] == [ + None, + self.to_reference_unit(3.0), + self.to_reference_unit(5.0), + ] + assert data[1]["totals"] == self.to_reference_unit(4.0) + + @with_feature("organizations:ddm-metrics-api-unit-normalization") + def test_groupby_using_project_id_does_not_get_demodulated_into_project(self) -> None: + self.new_project = self.create_project(name="Bar Again") + for value, transaction, platform, env, time in ( + (1, "/hello", "android", "prod", self.now()), + (3, "/hello", "android", "prod", self.now()), + (5, "/hello", "android", "prod", self.now()), + (2, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (5, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (8, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + ): + self.store_metric( + self.new_project.organization.id, + self.new_project.id, + "distribution", + TransactionMRI.DURATION.value, + { + "transaction": transaction, + "platform": platform, + "environment": env, + }, + self.ts(time), + value, + UseCaseID.TRANSACTIONS, + ) + + mql = self.mql("avg", TransactionMRI.DURATION.value, group_by="project_id") + query = MQLQuery(mql) + + results = self.run_query( + mql_queries=[query], + start=self.now() - timedelta(minutes=30), + end=self.now() + timedelta(hours=1, minutes=30), + interval=3600, + organization=self.project.organization, + projects=[self.project, self.new_project], + environments=[], + referrer="metrics.data.api", + ) + data = results["data"][0] + assert len(data) == 2 + data = sorted(data, key=lambda x: x["by"]["project_id"]) + assert data[0]["by"] == {"project_id": self.project.id} + assert data[1]["by"] == {"project_id": self.new_project.id} + assert data[1]["series"] == [ + None, + self.to_reference_unit(3.0), + self.to_reference_unit(5.0), + ] + assert data[1]["totals"] == self.to_reference_unit(4.0) + + @with_feature("organizations:ddm-metrics-api-unit-normalization") + def test_only_specific_queries_get_modulated_and_demodulated(self) -> None: + self.new_project = self.create_project(name="Bar Again") + for value, transaction, platform, env, time in ( + (1, "/hello", "android", "prod", self.now()), + (3, "/hello", "android", "prod", self.now()), + (5, "/hello", "android", "prod", self.now()), + (2, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (5, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + (8, "/hello", "android", "prod", self.now() + timedelta(hours=1, minutes=30)), + ): + self.store_metric( + self.new_project.organization.id, + self.new_project.id, + "distribution", + TransactionMRI.DURATION.value, + { + "transaction": transaction, + "platform": platform, + "environment": env, + }, + self.ts(time), + value, + UseCaseID.TRANSACTIONS, + ) + + mql_1 = self.mql("avg", TransactionMRI.DURATION.value, group_by="project_id") + mql_2 = self.mql("avg", TransactionMRI.DURATION.value, group_by="project") + query_1 = MQLQuery(mql_1) + query_2 = MQLQuery(mql_2) + + results = self.run_query( + mql_queries=[query_1, query_2], + start=self.now() - timedelta(minutes=30), + end=self.now() + timedelta(hours=1, minutes=30), + interval=3600, + organization=self.project.organization, + projects=[self.project, self.new_project], + environments=[], + referrer="metrics.data.api", + ) + data_1 = results["data"][0] + data_1 = sorted(data_1, key=lambda x: x["by"]["project_id"]) + data_2 = results["data"][1] + data_2 = sorted(data_2, key=lambda x: x["by"]["project"]) + + assert data_1[0]["by"] == {"project_id": self.project.id} + assert data_1[1]["by"] == {"project_id": self.new_project.id} + assert data_2[0]["by"] == {"project": self.project.slug} + assert data_2[1]["by"] == {"project": self.new_project.slug}