Skip to content

Commit cc1b347

Browse files
scholtzanscholtzan
authored
Merge branch 'main' into cache-stable-schemas
2 parents 988efe4 + 91ff526 commit cc1b347

File tree

10 files changed

+245
-0
lines changed

10 files changed

+245
-0
lines changed

‎.circleci/workflows.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,11 @@ jobs:
416416
command: |
417417
rm ~/telemetry-airflow/dags/* -f || true
418418
cp -a /tmp/workspace/generated-sql/dags/. ~/telemetry-airflow/dags/
419+
- restore_cache:
420+
keys:
421+
- telemetry-airflow-deps-v1-{{ checksum "~/telemetry-airflow/requirements.txt" }}-{{ checksum "~/telemetry-airflow/requirements-dev.txt" }}-{{ checksum "~/telemetry-airflow/requirements-override.txt" }}
422+
- telemetry-airflow-deps-v1-{{ checksum "~/telemetry-airflow/requirements.txt" }}-{{ checksum "~/telemetry-airflow/requirements-dev.txt" }}-
423+
- telemetry-airflow-deps-v1-
419424
- run:
420425
name: Install telemetry-airflow dependencies
421426
command: |
@@ -425,6 +430,10 @@ jobs:
425430
pip install -r requirements.txt
426431
pip install -r requirements-dev.txt
427432
pip install -r requirements-override.txt
433+
- save_cache:
434+
paths:
435+
- ~/telemetry-airflow/.venv
436+
key: telemetry-airflow-deps-v1-{{ checksum "~/telemetry-airflow/requirements.txt" }}-{{ checksum "~/telemetry-airflow/requirements-dev.txt" }}-{{ checksum "~/telemetry-airflow/requirements-override.txt" }}
428437
- run:
429438
name: 🧪 Test valid DAGs
430439
command: |

‎bqetl_project.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,3 +652,4 @@ retention_exclusion_list:
652652
- sql/moz-fx-data-shared-prod/firefox_desktop_background_update/baseline_clients_city_seen_v1
653653
- sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent/baseline_clients_city_seen_v1
654654
- sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks/baseline_clients_city_seen_v1
655+
- sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1

‎sql/moz-fx-data-shared-prod/ads_dap_derived/incrementality_v1/metadata.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,7 @@ description: |-
66
owners:
77
- [email protected]
88
- [email protected]
9+
workgroup_access:
10+
- role: roles/bigquery.dataEditor
11+
members:
12+
- workgroup:ads

‎sql/moz-fx-data-shared-prod/firefox_desktop/metrics_clients_first_seen/metadata.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
friendly_name: Metrics Clients First Seen
2+
description: |-
3+
Records the attributes first seen for each client from the metric ping.
4+
owners:
5+
- [email protected]
6+
labels:
7+
owner: kwindau

‎sql/moz-fx-data-shared-prod/firefox_desktop/metrics_clients_first_seen/view.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
CREATE OR REPLACE VIEW
2+
`moz-fx-data-shared-prod.firefox_desktop.metrics_clients_first_seen`
3+
AS
4+
SELECT
5+
*
6+
FROM
7+
`moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_first_seen_v1`

‎sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_daily_v1/schema.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,15 @@ fields:
1515
- name: normalized_channel
1616
type: STRING
1717
mode: NULLABLE
18+
description: The normalized channel the application is being distributed on.
1819
- name: n_metrics_ping
1920
type: INTEGER
2021
mode: NULLABLE
22+
description: Number of metrics pings we received from this client on this submission date.
2123
- name: days_sent_metrics_ping_bits
2224
type: INTEGER
2325
mode: NULLABLE
26+
description: Value of 1 if the client ID sent a metric ping on this date.
2427
- name: profile_group_id
2528
type: STRING
2629
mode: NULLABLE
@@ -44,6 +47,7 @@ fields:
4447
- name: default_search_engine
4548
type: STRING
4649
mode: NULLABLE
50+
description: The telemetry id of the search engine.
4751
- name: xpcom_abi
4852
type: STRING
4953
mode: NULLABLE

‎sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/metadata.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
friendly_name: Metrics Clients First Seen
2+
description: |-
3+
Captures the attributes seen on the metrics pings received on the
4+
date of the first received metrics ping for each client.
5+
owners:
6+
- [email protected]
7+
labels:
8+
schedule: daily
9+
dag: bqetl_glean_usage
10+
owner1: kwindau
11+
scheduling:
12+
dag_name: bqetl_glean_usage
13+
depends_on_past: true
14+
date_partition_parameter: null
15+
parameters:
16+
- submission_date:DATE:{{ds}}
17+
bigquery:
18+
time_partitioning:
19+
type: day
20+
field: first_seen_date
21+
require_partition_filter: false
22+
expiration_days: null
23+
range_partitioning: null
24+
clustering:
25+
fields:
26+
- sample_id
27+
require_column_descriptions: true

‎sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/query.sql‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
WITH _current AS (
2+
SELECT
3+
submission_date AS first_seen_date,
4+
submission_date,
5+
client_id,
6+
sample_id,
7+
normalized_channel,
8+
n_metrics_ping,
9+
days_sent_metrics_ping_bits,
10+
profile_group_id,
11+
search_with_ads_count_all,
12+
search_count_all,
13+
ad_clicks_count_all,
14+
apple_model_id,
15+
default_search_engine,
16+
xpcom_abi,
17+
installation_first_seen_admin_user,
18+
installation_first_seen_default_path,
19+
installation_first_seen_failure_reason,
20+
installation_first_seen_from_msi,
21+
installation_first_seen_install_existed,
22+
installation_first_seen_installer_type,
23+
installation_first_seen_other_inst,
24+
installation_first_seen_other_msix_inst,
25+
installation_first_seen_profdir_existed,
26+
installation_first_seen_silent,
27+
installation_first_seen_version
28+
FROM
29+
`moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_daily_v1`
30+
WHERE
31+
submission_date = @submission_date
32+
AND client_id IS NOT NULL
33+
),
34+
_previous AS (
35+
SELECT
36+
first_seen_date,
37+
submission_date,
38+
client_id,
39+
sample_id,
40+
normalized_channel,
41+
n_metrics_ping,
42+
days_sent_metrics_ping_bits,
43+
profile_group_id,
44+
search_with_ads_count_all,
45+
search_count_all,
46+
ad_clicks_count_all,
47+
apple_model_id,
48+
default_search_engine,
49+
xpcom_abi,
50+
installation_first_seen_admin_user,
51+
installation_first_seen_default_path,
52+
installation_first_seen_failure_reason,
53+
installation_first_seen_from_msi,
54+
installation_first_seen_install_existed,
55+
installation_first_seen_installer_type,
56+
installation_first_seen_other_inst,
57+
installation_first_seen_other_msix_inst,
58+
installation_first_seen_profdir_existed,
59+
installation_first_seen_silent,
60+
installation_first_seen_version
61+
FROM
62+
`moz-fx-data-shared-prod.firefox_desktop_derived.metrics_clients_first_seen_v1`
63+
WHERE
64+
submission_date < @submission_date
65+
)
66+
SELECT
67+
IF(
68+
_previous.client_id IS NULL
69+
OR _previous.first_seen_date >= _current.first_seen_date,
70+
_current,
71+
_previous
72+
).*
73+
FROM
74+
_current
75+
FULL JOIN
76+
_previous
77+
USING (client_id)

‎sql/moz-fx-data-shared-prod/firefox_desktop_derived/metrics_clients_first_seen_v1/schema.yaml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
fields:
2+
- name: first_seen_date
3+
type: DATE
4+
mode: NULLABLE
5+
description: The date when the telemetry ping was first received on the server side.
6+
- name: submission_date
7+
type: DATE
8+
mode: NULLABLE
9+
description: The date when the telemetry ping is received on the server side.
10+
- name: client_id
11+
type: STRING
12+
mode: NULLABLE
13+
description: A unique identifier (UUID) for the client.
14+
- name: sample_id
15+
type: INTEGER
16+
mode: NULLABLE
17+
description: A number, 0-99, that samples by client_id and allows filtering data
18+
for analysis. It is a pipeline-generated artifact that should match between pings.
19+
- name: normalized_channel
20+
type: STRING
21+
mode: NULLABLE
22+
description: The normalized channel the application is being distributed on.
23+
- name: n_metrics_ping
24+
type: INTEGER
25+
mode: NULLABLE
26+
description: Number of metrics pings we received from this client on this submission date.
27+
- name: days_sent_metrics_ping_bits
28+
type: INTEGER
29+
mode: NULLABLE
30+
description: Value of 1 if the client ID sent a metric ping on this date.
31+
- name: profile_group_id
32+
type: STRING
33+
mode: NULLABLE
34+
description: A UUID uniquely identifying the profile group, not shared with other telemetry data.
35+
- name: search_with_ads_count_all
36+
type: INTEGER
37+
mode: NULLABLE
38+
description: Number of searches with ads on client's last seen date in last 28 days.
39+
- name: search_count_all
40+
type: INTEGER
41+
mode: NULLABLE
42+
description: Number of searches on client's last seen date in last 28 days.
43+
- name: ad_clicks_count_all
44+
type: INTEGER
45+
mode: NULLABLE
46+
description: Number of ad clicks on client's last seen date in last 28 days.
47+
- name: apple_model_id
48+
type: STRING
49+
mode: NULLABLE
50+
description: The hw.model of the hardware running this software. Mac only. e.g. "MacBookPro14,1"
51+
- name: default_search_engine
52+
type: STRING
53+
mode: NULLABLE
54+
description: The telemetry id of the search engine.
55+
- name: xpcom_abi
56+
type: STRING
57+
mode: NULLABLE
58+
description: |-
59+
A string tag identifying the binary ABI of the current processor and compiler vtable. This is taken from the TARGET_XPCOM_ABI configure variable.
60+
It may not be available on all platforms, especially unusual processor or compiler combinations.
61+
For example: x86-msvc, ppc-gcc3
62+
This value should almost always be used in combination with the operating system.
63+
- name: installation_first_seen_admin_user
64+
type: BOOLEAN
65+
mode: NULLABLE
66+
description: Whether the installer is running from an elevated admin user.
67+
- name: installation_first_seen_default_path
68+
type: BOOLEAN
69+
mode: NULLABLE
70+
description: Whether the default path was used; optional, present if installer_type is "full"
71+
- name: installation_first_seen_failure_reason
72+
type: STRING
73+
mode: NULLABLE
74+
description: |-
75+
Only sent if unable to collect firstSeen data.
76+
Can have value "NotFoundError" if file not found or other values depending on the failure reason.
77+
- name: installation_first_seen_from_msi
78+
type: BOOLEAN
79+
mode: NULLABLE
80+
description: Whether this was an MSI install; optional, present if installer_type is "full".
81+
- name: installation_first_seen_install_existed
82+
type: BOOLEAN
83+
mode: NULLABLE
84+
description: Whether there was already an install in this location.
85+
- name: installation_first_seen_installer_type
86+
type: STRING
87+
mode: NULLABLE
88+
description: The type of installer used to install Firefox. The value is one of "stub", "full", or "msix".
89+
- name: installation_first_seen_other_inst
90+
type: BOOLEAN
91+
mode: NULLABLE
92+
description: Whether there was already any non-MSIX install on this system.
93+
- name: installation_first_seen_other_msix_inst
94+
type: BOOLEAN
95+
mode: NULLABLE
96+
description: Whether there was already any MSIX install on this system.
97+
- name: installation_first_seen_profdir_existed
98+
type: BOOLEAN
99+
mode: NULLABLE
100+
description: Whether the top-level profile directory existed.
101+
- name: installation_first_seen_silent
102+
type: BOOLEAN
103+
mode: NULLABLE
104+
description: Whether this was a silent install; optional, present if installer_type is "full".
105+
- name: installation_first_seen_version
106+
type: STRING
107+
mode: NULLABLE
108+
description: The application version installed by the installer (not necessarily the current version)

‎sql/moz-fx-data-shared-prod/shepherd/dataset_metadata.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ workgroup_access:
1212
- role: roles/bigquery.dataViewer
1313
members:
1414
- workgroup:ads
15+
- workgroup:ads/external-tapclicks

0 commit comments

Comments
 (0)