feat(DENG-9980): add enterprise metrics clients legacy query

[kik-kik]

feat(DENG-9980): add enterprise metrics clients legacy query

[dataops-ci-bot]

Integration report for "fix: update app_version fields to match upstream type"

sql.diff

Click to expand!

diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_enterprise.py /tmp/workspace/generated-sql/dags/bqetl_firefox_enterprise.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_enterprise.py	2025-10-30 09:03:03.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_firefox_enterprise.py	2025-10-30 09:05:41.000000000 +0000
@@ -105,6 +105,32 @@
         pool="DATA_ENG_EXTERNALTASKSENSOR",
     )
 
+    wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor(
+        task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2",
+        external_dag_id="bqetl_main_summary",
+        external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2",
+        execution_delta=datetime.timedelta(seconds=14400),
+        check_existence=True,
+        mode="reschedule",
+        poke_interval=datetime.timedelta(minutes=5),
+        allowed_states=ALLOWED_STATES,
+        failed_states=FAILED_STATES,
+        pool="DATA_ENG_EXTERNALTASKSENSOR",
+    )
+
+    wait_for_copy_deduplicate_main_ping = ExternalTaskSensor(
+        task_id="wait_for_copy_deduplicate_main_ping",
+        external_dag_id="copy_deduplicate",
+        external_task_id="copy_deduplicate_main_ping",
+        execution_delta=datetime.timedelta(seconds=18000),
+        check_existence=True,
+        mode="reschedule",
+        poke_interval=datetime.timedelta(minutes=5),
+        allowed_states=ALLOWED_STATES,
+        failed_states=FAILED_STATES,
+        pool="DATA_ENG_EXTERNALTASKSENSOR",
+    )
+
     firefox_desktop_derived__enterprise_metrics__v1 = bigquery_etl_query(
         task_id="firefox_desktop_derived__enterprise_metrics__v1",
         destination_table="enterprise_metrics_v1",
@@ -127,6 +153,17 @@
         depends_on_past=False,
     )
 
+    firefox_desktop_derived__enterprise_metrics_clients_legacy__v1 = bigquery_etl_query(
+        task_id="firefox_desktop_derived__enterprise_metrics_clients_legacy__v1",
+        destination_table="enterprise_metrics_clients_legacy_v1",
+        dataset_id="firefox_desktop_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+    )
+
     firefox_desktop_derived__enterprise_metrics__v1.set_upstream(
         firefox_desktop_derived__enterprise_metrics_clients__v1
     )
@@ -146,3 +183,11 @@
     firefox_desktop_derived__enterprise_metrics_clients__v1.set_upstream(
         wait_for_copy_deduplicate_all
     )
+
+    firefox_desktop_derived__enterprise_metrics_clients_legacy__v1.set_upstream(
+        wait_for_checks__fail_telemetry_derived__clients_last_seen__v2
+    )
+
+    firefox_desktop_derived__enterprise_metrics_clients_legacy__v1.set_upstream(
+        wait_for_copy_deduplicate_main_ping
+    )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py /tmp/workspace/generated-sql/dags/bqetl_main_summary.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py	2025-10-30 09:02:59.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_main_summary.py	2025-10-30 09:05:33.000000000 +0000
@@ -149,6 +149,13 @@
         )
 
         ExternalTaskMarker(
+            task_id="bqetl_firefox_enterprise__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2",
+            external_dag_id="bqetl_firefox_enterprise",
+            external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2",
+            execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=72000)).isoformat() }}",
+        )
+
+        ExternalTaskMarker(
             task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2",
             external_dag_id="bqetl_desktop_conv_evnt_categorization",
             external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2",
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop: enterprise_metrics_clients_legacy
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived: enterprise_metrics_clients_legacy_v1
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients/view.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients/view.sql	2025-10-30 08:59:11.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients/view.sql	2025-10-30 08:55:28.000000000 +0000
@@ -4,6 +4,11 @@
 SELECT
   *,
   CASE
+    WHEN app_version_major >= 144
+      AND normalized_channel = "esr"
+      AND policies_count > 1
+      AND distribution_id IS NULL
+      THEN "enterprise_esr"
     WHEN normalized_channel = "release"
       AND ((policies_count > 1) OR (policies_is_enterprise = TRUE))
       THEN "enterprise_release"
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/metadata.yaml	2025-10-30 08:58:39.000000000 +0000
@@ -0,0 +1,14 @@
+friendly_name: Enterprise Metrics Clients Legacy
+description: |-
+  Please provide a description for the query
+owners: []
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  view.sql:
+  - moz-fx-data-shared-prod.firefox_desktop_derived.enterprise_metrics_clients_legacy_v1
+require_column_descriptions: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/view.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/view.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/enterprise_metrics_clients_legacy/view.sql	2025-10-30 08:55:28.000000000 +0000
@@ -0,0 +1,39 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.enterprise_metrics_clients_legacy`
+AS
+SELECT
+  *,
+  CASE
+    WHEN `mozfun.norm.browser_version_info`(app_version).major_version >= 144
+      AND normalized_channel = "esr"
+      AND policies_count > 1
+      AND distribution_id IS NULL
+      THEN "enterprise_esr"
+    WHEN normalized_channel = "release"
+      AND ((policies_count > 1) OR (policies_is_enterprise = TRUE))
+      THEN "enterprise_release"
+    WHEN normalized_channel = "release"
+      AND (
+        (distribution_id IS NOT NULL)
+        OR (policies_count = 0)
+        OR (policies_is_enterprise = FALSE)
+      )
+      THEN "consumer_release"
+    WHEN normalized_channel = "esr"
+      AND ((policies_count > 1) AND (distribution_id IS NULL))
+      THEN "enterprise_esr"
+    WHEN normalized_channel = "esr"
+      AND (
+        (distribution_id IS NOT NULL)
+        OR (policies_count = 0)
+        OR (policies_is_enterprise = FALSE)
+      )
+      THEN "consumer_esr"
+    WHEN normalized_channel = "release"
+      THEN "unknown_release"
+    WHEN normalized_channel = "esr"
+      THEN "unknown_esr"
+    ELSE "unexpected_classification" -- TODO: we should set up an alert for this, but not fail the query.
+  END AS enterprise_classification,
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.enterprise_metrics_clients_legacy_v1`
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/metadata.yaml	2025-10-30 08:58:41.000000000 +0000
@@ -0,0 +1,35 @@
+friendly_name: Enterprise Metrics (clients) - Legacy
+description: |-
+  Enterprise Metrics (clients) based on our legacy telemetry.
+owners:
+- [email protected]
+- [email protected]
+labels:
+  incremental: true
+  table_type: client_level
+  schedule: daily
+  dag: bqetl_firefox_enterprise
+  owner1: pissac
+  owner2: kik
+scheduling:
+  dag_name: bqetl_firefox_enterprise
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: 775.0
+  range_partitioning: null
+  clustering:
+    fields:
+    - normalized_channel
+    - is_dau
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.telemetry.desktop_active_users
+  - moz-fx-data-shared-prod.telemetry.main
+require_column_descriptions: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/query.sql	2025-10-30 08:55:28.000000000 +0000
@@ -0,0 +1,61 @@
+WITH daily_users AS (
+  SELECT
+    submission_date,
+    client_id,
+    sample_id,
+    normalized_channel,
+    app_version,
+    is_desktop,
+    is_dau,
+  FROM
+    `moz-fx-data-shared-prod.telemetry.desktop_active_users`
+  WHERE
+    submission_date = @submission_date
+      -- enterprise is always only "esr" or "release" channels
+    AND normalized_channel IN ("release", "esr")
+    AND is_daily_user
+),
+most_recent_client_distribution_and_policy_metrics AS (
+  SELECT
+    client_id,
+    normalized_channel,
+    ARRAY_AGG(
+      payload.processes.parent.scalars.policies_count IGNORE NULLS
+      ORDER BY
+        submission_timestamp DESC
+    )[SAFE_OFFSET(0)] AS policies_count,
+    ARRAY_AGG(
+      payload.processes.parent.scalars.policies_is_enterprise IGNORE NULLS
+      ORDER BY
+        submission_timestamp DESC
+    )[SAFE_OFFSET(0)] AS policies_is_enterprise,
+    ARRAY_AGG(environment.partner.distribution_id IGNORE NULLS ORDER BY submission_timestamp DESC)[
+      SAFE_OFFSET(0)
+    ] AS distribution_id,
+  FROM
+    `moz-fx-data-shared-prod.telemetry.main`
+  WHERE
+    DATE(submission_timestamp)
+    BETWEEN DATE_SUB(@submission_date, INTERVAL 27 DAY)
+    AND @submission_date
+    -- enterprise is always only "esr" or "release" channels
+    AND normalized_channel IN ("release", "esr")
+  GROUP BY
+    ALL
+)
+SELECT
+  @submission_date AS submission_date,
+  client_id,
+  sample_id,
+  normalized_channel,
+  app_version,
+  distribution_id,
+  is_dau,
+  is_desktop,
+  policies_count,
+  policies_is_enterprise,
+FROM
+  daily_users
+INNER JOIN
+  most_recent_client_distribution_and_policy_metrics
+  USING (client_id, normalized_channel)
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/schema.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_legacy_v1/schema.yaml	2025-10-30 08:55:28.000000000 +0000
@@ -0,0 +1,63 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: |
+    Logical date corresponding to the partition (date when our server received the ping)
+    that was processed for generating the metrics.
+
+- mode: NULLABLE
+  name: client_id
+  type: STRING
+  description: |
+    A UUID uniquely identifying the client.
+
+- mode: NULLABLE
+  name: sample_id
+  type: INTEGER
+  description: |
+    Hashed version of client_id (if present) useful for partitioning; ranges from 0 to 99
+
+- mode: NULLABLE
+  name: normalized_channel
+  type: STRING
+  description: |
+    Normalized channel the application is being distributed on. For example, release, beta, etc.
+
+- mode: NULLABLE
+  name: app_version
+  type: STRING
+  description: |
+    User visible version string (e.g. "1.0.3") for the browser.
+
+- mode: NULLABLE
+  name: distribution_id
+  type: STRING
+  description: |
+    The distribution id associated with the install of Firefox.
+
+- mode: NULLABLE
+  name: is_dau
+  type: BOOLEAN
+  description: |
+    Indicates whether the client met our definition of dau on the day.
+
+- mode: NULLABLE
+  name: is_desktop
+  type: BOOLEAN
+  description: |
+    Indicates whether the client met conditions to be counted as Desktop DAU on the day.
+
+- mode: NULLABLE
+  name: policies_count
+  type: INTEGER
+  description: |
+    A uint with the number of active enterprise policies, collected once at startup.
+    This metric was generated to correspond to the Legacy Telemetry scalar policies.count.
+
+- mode: NULLABLE
+  name: policies_is_enterprise
+  type: BOOLEAN
+  description: |
+    Attempt to determine if the user is an enterprise user based on various signals.
+    This metric was generated to correspond to the Legacy Telemetry scalar policies.is_enterprise
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/metadata.yaml	2025-10-30 08:59:12.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/metadata.yaml	2025-10-30 08:58:41.000000000 +0000
@@ -1,6 +1,6 @@
-friendly_name: Enterprise Metrics (clients)
+friendly_name: Enterprise Metrics (clients) - Glean
 description: |-
-  Enterprise Metrics (clients).
+  Enterprise Metrics (clients) based on Glean telemetry.
 owners:
 - [email protected]
 - [email protected]
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/query.sql	2025-10-30 08:59:12.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/query.sql	2025-10-30 08:55:28.000000000 +0000
@@ -4,10 +4,16 @@
       submission_date,
       client_id,
       sample_id,
+      legacy_telemetry_client_id,
       channel,
       distribution_id,
       is_daily_user,
       is_dau,
+      is_desktop,
+      app_version,
+      app_version_major,
+      app_version_minor,
+      app_version_patch_revision,
     FROM
       `moz-fx-data-shared-prod.firefox_desktop.baseline_active_users`
     WHERE
@@ -32,8 +38,14 @@
     SELECT
       client_id,
       sample_id,
+      legacy_telemetry_client_id,
       channel,
       is_dau,
+      is_desktop,
+      app_version,
+      app_version_major,
+      app_version_minor,
+      app_version_patch_revision,
     FROM
       active_users_base
     WHERE
@@ -43,9 +55,15 @@
   SELECT
     client_id,
     sample_id,
+    legacy_telemetry_client_id,
     channel AS normalized_channel,
     distribution_id,
     is_dau,
+    is_desktop,
+    app_version,
+    app_version_major,
+    app_version_minor,
+    app_version_patch_revision,
   FROM
     daily_users
   LEFT JOIN
@@ -79,9 +97,15 @@
   sample_id,
   normalized_channel,
   distribution_id,
+  app_version,
+  app_version_major,
+  app_version_minor,
+  app_version_patch_revision,
   is_dau,
+  is_desktop,
   policies_count,
   policies_is_enterprise,
+  legacy_telemetry_client_id,
 FROM
   client_baseline
 LEFT JOIN
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/schema.yaml	2025-10-30 08:59:12.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/enterprise_metrics_clients_v1/schema.yaml	2025-10-30 08:55:28.000000000 +0000
@@ -31,12 +31,42 @@
     The distribution id associated with the install of Firefox.
 
 - mode: NULLABLE
+  name: app_version
+  type: STRING
+  description: |
+    User visible version string (e.g. "1.0.3") for the browser.
+
+- mode: NULLABLE
+  name: app_version_major
+  type: NUMERIC
+  description: |
+    Major version of the application (Firefox).
+
+- mode: NULLABLE
+  name: app_version_minor
+  type: NUMERIC
+  description: |
+    Minor version of the application (Firefox).
+
+- mode: NULLABLE
+  name: app_version_patch_revision
+  type: NUMERIC
+  description: |
+    Patch version of the application (Firefox).
+
+- mode: NULLABLE
   name: is_dau
   type: BOOLEAN
   description: |
     Indicates whether the client met our definition of dau on the day.
 
 - mode: NULLABLE
+  name: is_desktop
+  type: BOOLEAN
+  description: |
+    Indicates whether the client met conditions to be counted as Desktop DAU on the day.
+
+- mode: NULLABLE
   name: policies_count
   type: INTEGER
   description: |
@@ -49,3 +79,9 @@
   description: |
     Attempt to determine if the user is an enterprise user based on various signals.
     This metric was generated to correspond to the Legacy Telemetry scalar policies.is_enterprise
+
+- mode: NULLABLE
+  name: legacy_telemetry_client_id
+  type: STRING
+  description: |
+    A unique identifier (UUID) for the client in legacy telemetry.

Link to full diff