feat(DENG-10038): Create firefox_desktop_derived.metrics_clients_first_seen_v1

bqetl_project.yaml

@@ -652,3 +652,4 @@ retention_exclusion_list:
 - sql/moz-fx-data-shared-prod/firefox_desktop_background_update/baseline_clients_city_seen_v1
 - sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent/baseline_clients_city_seen_v1
 - sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks/baseline_clients_city_seen_v1
+- sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1

sql/moz-fx-data-shared-prod/firefox_desktop/metrics_clients_first_seen/metadata.yaml

@@ -0,0 +1,7 @@
+friendly_name: Metrics Clients First Seen
+description: |-
+  Records the attributes first seen for each client from the metric ping.
+owners:
+- [email protected]
+labels:
+  owner: kwindau

sql/moz-fx-data-shared-prod/firefox_desktop/metrics_clients_first_seen/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.metrics_clients_first_seen`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_first_seen_v1`

sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_daily_v1/schema.yaml

@@ -15,12 +15,15 @@ fields:
 - name: normalized_channel
   type: STRING
   mode: NULLABLE
+  description: The normalized channel the application is being distributed on.
 - name: n_metrics_ping
   type: INTEGER
   mode: NULLABLE
+  description: Number of metrics pings we received from this client on this submission date.
 - name: days_sent_metrics_ping_bits
   type: INTEGER
   mode: NULLABLE
+  description: Value of 1 if the client ID sent a metric ping on this date.
 - name: profile_group_id
   type: STRING
   mode: NULLABLE
@@ -44,6 +47,7 @@ fields:
 - name: default_search_engine
   type: STRING
   mode: NULLABLE
+  description: The telemetry id of the search engine.
 - name: xpcom_abi
   type: STRING
   mode: NULLABLE

sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/metadata.yaml

@@ -0,0 +1,27 @@
+friendly_name: Metrics Clients First Seen
+description: |-
+  Captures the attributes seen on the metrics pings received on the
+  date of the first received metrics ping for each client.
+owners:
+- [email protected]
+labels:
+  schedule: daily
+  dag: bqetl_glean_usage
+  owner1: kwindau
+scheduling:
+  dag_name: bqetl_glean_usage
+  depends_on_past: true
+  date_partition_parameter: null
+  parameters:
+  - submission_date:DATE:{{ds}}
+bigquery:
+  time_partitioning:
+    type: day
+    field: first_seen_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - sample_id
+require_column_descriptions: true

sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/query.sql

@@ -0,0 +1,77 @@
+WITH _current AS (
+  SELECT
+    submission_date AS first_seen_date,
+    submission_date,
+    client_id,
+    sample_id,
+    normalized_channel,
+    n_metrics_ping,
+    days_sent_metrics_ping_bits,
+    profile_group_id,
+    search_with_ads_count_all,
+    search_count_all,
+    ad_clicks_count_all,
+    apple_model_id,
+    default_search_engine,
+    xpcom_abi,
+    installation_first_seen_admin_user,
+    installation_first_seen_default_path,
+    installation_first_seen_failure_reason,
+    installation_first_seen_from_msi,
+    installation_first_seen_install_existed,
+    installation_first_seen_installer_type,
+    installation_first_seen_other_inst,
+    installation_first_seen_other_msix_inst,
+    installation_first_seen_profdir_existed,
+    installation_first_seen_silent,
+    installation_first_seen_version
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_daily_v1`
+  WHERE
+    submission_date = @submission_date
+    AND client_id IS NOT NULL
+),
+_previous AS (
+  SELECT
+    first_seen_date,
+    submission_date,
+    client_id,
+    sample_id,
+    normalized_channel,
+    n_metrics_ping,
+    days_sent_metrics_ping_bits,
+    profile_group_id,
+    search_with_ads_count_all,
+    search_count_all,
+    ad_clicks_count_all,
+    apple_model_id,
+    default_search_engine,
+    xpcom_abi,
+    installation_first_seen_admin_user,
+    installation_first_seen_default_path,
+    installation_first_seen_failure_reason,
+    installation_first_seen_from_msi,
+    installation_first_seen_install_existed,
+    installation_first_seen_installer_type,
+    installation_first_seen_other_inst,
+    installation_first_seen_other_msix_inst,
+    installation_first_seen_profdir_existed,
+    installation_first_seen_silent,
+    installation_first_seen_version
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_first_seen_v1`
+  WHERE
+    submission_date < @submission_date
+)
+SELECT
+  IF(
+    _previous.client_id IS NULL
+    OR _previous.first_seen_date >= _current.first_seen_date,
+    _current,
+    _previous
+  ).*
+FROM
+  _current
+FULL JOIN
+  _previous
+  USING (client_id)

sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/schema.yaml

@@ -0,0 +1,108 @@
+fields:
+- name: first_seen_date
+  type: DATE
+  mode: NULLABLE
+  description: The date when the telemetry ping was first received on the server side.
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: The date when the telemetry ping is received on the server side.
+- name: client_id
+  type: STRING
+  mode: NULLABLE
+  description: A unique identifier (UUID) for the client.
+- name: sample_id
+  type: INTEGER
+  mode: NULLABLE
+  description: A number, 0-99, that samples by client_id and allows filtering data
+    for analysis. It is a pipeline-generated artifact that should match between pings.
+- name: normalized_channel
+  type: STRING
+  mode: NULLABLE
+  description: The normalized channel the application is being distributed on.
+- name: n_metrics_ping
+  type: INTEGER
+  mode: NULLABLE
+  description: Number of metrics pings we received from this client on this submission date.
+- name: days_sent_metrics_ping_bits
+  type: INTEGER
+  mode: NULLABLE
+  description: Value of 1 if the client ID sent a metric ping on this date.
+- name: profile_group_id
+  type: STRING
+  mode: NULLABLE
+  description: A UUID uniquely identifying the profile group, not shared with other telemetry data.
+- name: search_with_ads_count_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Number of searches with ads on client's last seen date in last 28 days.
+- name: search_count_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Number of searches on client's last seen date in last 28 days.
+- name: ad_clicks_count_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Number of ad clicks on client's last seen date in last 28 days.
+- name: apple_model_id
+  type: STRING
+  mode: NULLABLE
+  description: The hw.model of the hardware running this software. Mac only. e.g. "MacBookPro14,1"
+- name: default_search_engine
+  type: STRING
+  mode: NULLABLE
+  description: The telemetry id of the search engine.
+- name: xpcom_abi
+  type: STRING
+  mode: NULLABLE
+  description: |-
+    A string tag identifying the binary ABI of the current processor and compiler vtable. This is taken from the TARGET_XPCOM_ABI configure variable.
+    It may not be available on all platforms, especially unusual processor or compiler combinations.
+    For example: x86-msvc, ppc-gcc3
+    This value should almost always be used in combination with the operating system.
+- name: installation_first_seen_admin_user
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the installer is running from an elevated admin user.
+- name: installation_first_seen_default_path
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the default path was used; optional, present if installer_type is "full"
+- name: installation_first_seen_failure_reason
+  type: STRING
+  mode: NULLABLE
+  description: |-
+    Only sent if unable to collect firstSeen data.
+    Can have value "NotFoundError" if file not found or other values depending on the failure reason.
+- name: installation_first_seen_from_msi
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether this was an MSI install; optional, present if installer_type is "full".
+- name: installation_first_seen_install_existed
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether there was already an install in this location.
+- name: installation_first_seen_installer_type
+  type: STRING
+  mode: NULLABLE
+  description: The type of installer used to install Firefox. The value is one of "stub", "full", or "msix".
+- name: installation_first_seen_other_inst
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether there was already any non-MSIX install on this system.
+- name: installation_first_seen_other_msix_inst
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether there was already any MSIX install on this system.
+- name: installation_first_seen_profdir_existed
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the top-level profile directory existed.
+- name: installation_first_seen_silent
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether this was a silent install; optional, present if installer_type is "full".
+- name: installation_first_seen_version
+  type: STRING
+  mode: NULLABLE
+  description: The application version installed by the installer (not necessarily the current version)