Skip to content

Commit dbbac45

Browse files
kwindaukwindau
authored
Merge branch 'main' into DENG-10038-metrics-cfs
2 parents e6a4df4 + b3f6088 commit dbbac45

File tree

44 files changed

+1004
-300
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1004
-300
lines changed

‎sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ bigquery:
2727
field: date
2828
require_partition_filter: false
2929
monitoring:
30-
enabled: true
30+
enabled: false
31+
deprecated: true

‎sql/moz-fx-data-shared-prod/static/country_names_v1/metadata.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,7 @@ description: |-
55
66
Generated automatically by `sql_generators/country_code_lookup`; modify `aliases.yaml` to add a mapping to the
77
appropriate country code.
8+
workgroup_access:
9+
- role: roles/bigquery.dataViewer
10+
members:
11+
- workgroup:ads/external-tapclicks

‎sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_logical_subscriptions_history_v1/query.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ subscriptions_history_invoice_summaries AS (
126126
JOIN
127127
active_subscriptions_history AS history
128128
ON invoices.subscription_id = history.subscription.id
129-
AND invoices.created < history.valid_to
129+
AND COALESCE(invoices.status_transitions_finalized_at, invoices.created) < history.valid_to
130130
LEFT JOIN
131131
`moz-fx-data-shared-prod.stripe_external.charge_v1` AS charges
132132
ON invoices.id = charges.invoice_id

‎sql_generators/usage_reporting/templates/app_union.view.sql.jinja‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,12 @@ AS
77
UNION ALL
88
{% endif -%}
99
SELECT
10-
{% if view_name == "usage_reporting_clients_daily" %}
11-
{% if app_name == "firefox_desktop" %}
12-
"{{ channel.channel_dataset }}" AS normalized_app_id,
13-
`mozfun.norm.app_channel`(app_channel) AS normalized_channel,
14-
{% else %}
15-
"{{ channel.channel_dataset }}" AS normalized_app_id,
16-
"{{ channel.channel_name }}" AS normalized_channel,
17-
{% endif %}
10+
{% if app_name == "firefox_desktop" %}
11+
"{{ channel.channel_dataset }}" AS normalized_app_id,
12+
`mozfun.norm.app_channel`(app_channel) AS normalized_channel,
13+
{% else %}
14+
"{{ channel.channel_dataset }}" AS normalized_app_id,
15+
"{{ channel.channel_name }}" AS normalized_channel,
1816
{% endif %}
1917
*,
2018
FROM

‎sql_generators/usage_reporting/templates/usage_reporting_active_users.view.sql.jinja‎

Lines changed: 11 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,52 +2,17 @@
22
CREATE OR REPLACE VIEW
33
`{{ project_id }}.{{ app_name }}.{{ view_name }}`
44
AS
5-
WITH cls AS (
6-
SELECT
7-
submission_date,
5+
WITH first_seen AS (
6+
SELECT
87
usage_profile_id,
9-
{% if app_name == "focus_ios" -%}
10-
app_channel,
11-
{%- endif -%}
12-
days_seen_bits,
13-
days_active_bits,
14-
days_created_profile_bits
15-
FROM
16-
`{{ project_id }}.{{ app_name }}.usage_reporting_clients_last_seen`
17-
WHERE
18-
submission_date >= '2025-03-01'
19-
),
20-
cd AS (
21-
SELECT
22-
normalized_app_id,
23-
normalized_channel,
24-
submission_date,
25-
usage_profile_id,
26-
first_run_date,
27-
app_channel,
28-
normalized_country_code,
29-
os,
30-
os_version,
31-
app_build,
32-
app_display_version,
33-
distribution_id,
34-
is_default_browser,
35-
reason,
36-
is_active,
37-
{% if app_name == "firefox_desktop" -%}
38-
windows_build_number,
39-
{%- endif -%}
40-
FROM
41-
`{{ project_id }}.{{ app_name }}.usage_reporting_clients_daily`
42-
WHERE
43-
submission_date >= '2025-03-01'
8+
first_seen_date,
9+
FROM `{{ project_id }}.{{ app_name }}.usage_reporting_clients_first_seen`
4410
)
4511
SELECT
4612
* EXCEPT(submission_date, app_channel, normalized_country_code, app_display_version),
47-
last_seen.submission_date,
48-
daily.submission_date AS `date`,
49-
daily.normalized_channel AS channel,
50-
IFNULL(daily.normalized_country_code, "??") AS country,
13+
submission_date,
14+
normalized_channel AS channel,
15+
IFNULL(normalized_country_code, "??") AS country,
5116
EXTRACT(YEAR FROM first_seen.first_seen_date) AS first_seen_year,
5217
{% if app_name in ("fenix", "firefox_desktop") %}
5318
CASE
@@ -100,10 +65,9 @@ SELECT
10065
IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 7, FALSE) AS is_weekly_user,
10166
IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 28, FALSE) AS is_monthly_user,
10267
FROM
103-
cls AS last_seen
104-
LEFT JOIN
105-
cd AS daily
106-
USING(submission_date, usage_profile_id)
68+
`{{ project_id }}.{{ app_name }}.usage_reporting_clients_last_seen`
10769
LEFT JOIN
108-
`{{ project_id }}.{{ app_name }}.usage_reporting_clients_first_seen` AS first_seen
70+
first_seen
10971
USING (usage_profile_id)
72+
WHERE
73+
submission_date >= '2025-03-01'

‎sql_generators/usage_reporting/templates/usage_reporting_clients_daily_v1.query.sql.jinja‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,20 +42,20 @@ WITH usage_reporting_base AS (
4242
SELECT
4343
submission_date,
4444
usage_profile_id,
45-
udf.mode_last(ARRAY_AGG(first_run_date IGNORE NULLS ORDER BY submission_timestamp ASC)) AS first_run_date,
46-
udf.mode_last(ARRAY_AGG(app_channel IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_channel,
47-
udf.mode_last(ARRAY_AGG(normalized_country_code IGNORE NULLS ORDER BY submission_timestamp ASC)) AS normalized_country_code,
48-
udf.mode_last(ARRAY_AGG(os IGNORE NULLS ORDER BY submission_timestamp ASC)) AS os,
49-
udf.mode_last(ARRAY_AGG(os_version IGNORE NULLS ORDER BY submission_timestamp ASC)) AS os_version,
50-
udf.mode_last(ARRAY_AGG(app_build IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_build,
51-
udf.mode_last(ARRAY_AGG(app_display_version IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_display_version,
52-
udf.mode_last(ARRAY_AGG(distribution_id IGNORE NULLS ORDER BY submission_timestamp ASC)) AS distribution_id,
53-
udf.mode_last(ARRAY_AGG(is_default_browser IGNORE NULLS ORDER BY submission_timestamp ASC)) AS is_default_browser,
54-
udf.mode_last(ARRAY_AGG(reason IGNORE NULLS ORDER BY submission_timestamp ASC)) AS reason,
45+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(first_run_date IGNORE NULLS ORDER BY submission_timestamp ASC)) AS first_run_date,
46+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(app_channel IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_channel,
47+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(normalized_country_code IGNORE NULLS ORDER BY submission_timestamp ASC)) AS normalized_country_code,
48+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(os IGNORE NULLS ORDER BY submission_timestamp ASC)) AS os,
49+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(os_version IGNORE NULLS ORDER BY submission_timestamp ASC)) AS os_version,
50+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(app_build IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_build,
51+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(app_display_version IGNORE NULLS ORDER BY submission_timestamp ASC)) AS app_display_version,
52+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(distribution_id IGNORE NULLS ORDER BY submission_timestamp ASC)) AS distribution_id,
53+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(is_default_browser IGNORE NULLS ORDER BY submission_timestamp ASC)) AS is_default_browser,
54+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(reason IGNORE NULLS ORDER BY submission_timestamp ASC)) AS reason,
5555
-- is_active definition is different between desktop and mobile products.
5656
{% if app_name == "firefox_desktop" %}
5757
COALESCE(LOGICAL_OR(is_active), SUM(browser_engagement_uri_count) > 0 AND SUM(browser_engagement_active_ticks) > 0, FALSE) AS is_active,
58-
udf.mode_last(ARRAY_AGG(windows_build_number IGNORE NULLS ORDER BY submission_timestamp ASC)) AS windows_build_number,
58+
`moz-fx-data-shared-prod`.udf.mode_last(ARRAY_AGG(windows_build_number IGNORE NULLS ORDER BY submission_timestamp ASC)) AS windows_build_number,
5959
{% else %}
6060
COALESCE(LOGICAL_OR(is_active), SUM(IF(duration BETWEEN 0 AND 100000, duration, 0)) > 0, FALSE) AS is_active,
6161
{% endif %}

‎sql_generators/usage_reporting/templates/usage_reporting_clients_daily_v1.schema.yaml.jinja‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ fields:
99
- mode: NULLABLE
1010
name: usage_profile_id
1111
type: STRING
12-
description:
12+
description: |
13+
A UUID of the usage_profile.
1314

1415
- mode: NULLABLE
1516
name: first_run_date

‎sql_generators/usage_reporting/templates/usage_reporting_clients_first_seen_v1.query.sql.jinja‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22
WITH
33
_current AS (
44
SELECT
5-
usage_profile_id,
65
{% raw %}
76
{% if is_init() %}
87
MIN(submission_date) AS first_seen_date,
98
{% else %}
109
@submission_date AS first_seen_date,
1110
{% endif %}
1211
{% endraw %}
12+
* EXCEPT(submission_date, is_active),
1313
FROM
14-
`{{ project_id }}.{{ channel_dataset }}.usage_reporting_clients_daily`
14+
`{{ project_id }}.{{ channel_dataset }}_derived.usage_reporting_clients_daily_v1`
1515
WHERE
1616
usage_profile_id IS NOT NULL
1717
{% raw %}
@@ -40,8 +40,7 @@ _previous AS (
4040
)
4141

4242
SELECT
43-
first_seen_date,
44-
usage_profile_id,
43+
_current.*,
4544
FROM
4645
_current
4746
LEFT JOIN

‎sql_generators/usage_reporting/templates/usage_reporting_clients_first_seen_v1.schema.yaml.jinja‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,77 @@ fields:
1111
type: STRING
1212
description: |
1313
A UUID of the usage_profile.
14+
15+
- mode: NULLABLE
16+
name: first_run_date
17+
type: DATE
18+
description: |
19+
The date of the first run of the application.
20+
21+
- mode: NULLABLE
22+
name: app_channel
23+
type: STRING
24+
description: |
25+
The channel the application is being distributed on.
26+
27+
- mode: NULLABLE
28+
name: normalized_country_code
29+
type: STRING
30+
description: |
31+
Country code
32+
33+
- mode: NULLABLE
34+
name: os
35+
type: STRING
36+
description: |
37+
The name of the operating system.
38+
39+
- mode: NULLABLE
40+
name: os_version
41+
type: STRING
42+
description: |
43+
The user-visible version of the operating system (e.g. "1.2.3").
44+
If the version detection fails, this metric gets set to Unknown.
45+
46+
- mode: NULLABLE
47+
name: app_build
48+
type: STRING
49+
description: |
50+
The build identifier generated by the CI system (e.g. "1234/A").
51+
If the value was not provided through configuration, this metric gets set to Unknown.
52+
53+
- mode: NULLABLE
54+
name: app_display_version
55+
type: STRING
56+
description: |
57+
The user visible version string (e.g. "1.0.3").
58+
If the value was not provided through configuration, this metric gets set to Unknown.
59+
60+
- mode: NULLABLE
61+
name: distribution_id
62+
type: STRING
63+
description: |
64+
A string containing the distribution identifier. This was used to identify installs
65+
from Mozilla Online, but now also identifies partnership deal distributions.
66+
67+
- mode: NULLABLE
68+
name: is_default_browser
69+
type: BOOLEAN
70+
description: |
71+
Is Firefox the default browser.
72+
73+
- mode: NULLABLE
74+
name: reason
75+
type: STRING
76+
description: |
77+
The optional reason the ping was submitted. The specific values for reason are specific to each ping,
78+
and are documented in the ping's pings.yaml file.
79+
80+
{% if app_name == "firefox_desktop" -%}
81+
- mode: NULLABLE
82+
name: windows_build_number
83+
type: INTEGER
84+
description: |
85+
The optional Windows build number, reported by Windows (e.g. 22000) and not set for other platforms.
86+
{%- endif -%}
87+

‎sql_generators/usage_reporting/templates/usage_reporting_clients_last_seen_v1.query.sql.jinja‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,54 +3,74 @@
33
-- 28 days for each usage criterion as a single 64-bit integer.
44
WITH _current AS (
55
SELECT
6-
usage_profile_id,
76
-- The rightmost bit in 'days_since_seen' represents whether the user sent a usage_reporting ping on the submission_date.
87
CAST(TRUE AS INT64) AS days_seen_bits,
98
-- The rightmost bit in days_active_bits represents whether the user counts as active on the submission_date.
109
CAST(TRUE AS INT64) & CAST(is_active AS INT64) AS days_active_bits,
11-
udf.days_since_created_profile_as_28_bits(
10+
`moz-fx-data-shared-prod`.udf.days_since_created_profile_as_28_bits(
1211
DATE_DIFF(submission_date, first_run_date, DAY)
1312
) AS days_created_profile_bits,
14-
{% if channel_dataset == "org_mozilla_ios_focus" %}
15-
app_channel,
16-
{% endif %}
13+
usage_profile_id,
14+
app_channel,
15+
first_run_date,
16+
normalized_country_code,
17+
os,
18+
os_version,
19+
app_build,
20+
app_display_version,
21+
distribution_id,
22+
is_default_browser,
23+
reason,
24+
{% if app_name == "firefox_desktop" -%}
25+
windows_build_number,
26+
{%- endif -%}
1727
FROM
1828
`{{ project_id }}.{{ channel_dataset }}.usage_reporting_clients_daily`
1929
WHERE
2030
submission_date = @submission_date
2131
),
2232
_previous AS (
2333
SELECT
24-
usage_profile_id,
2534
days_seen_bits,
2635
days_active_bits,
2736
days_created_profile_bits,
28-
{% if channel_dataset == "org_mozilla_ios_focus" %}
29-
app_channel,
30-
{% endif %}
37+
usage_profile_id,
38+
app_channel,
39+
first_run_date,
40+
normalized_country_code,
41+
os,
42+
os_version,
43+
app_build,
44+
app_display_version,
45+
distribution_id,
46+
is_default_browser,
47+
reason,
48+
{% if app_name == "firefox_desktop" -%}
49+
windows_build_number,
50+
{%- endif -%}
3151
FROM
3252
`{{ project_id }}.{{ channel_dataset }}_derived.{{ view_name }}_v1`
3353
WHERE
3454
submission_date = DATE_SUB(@submission_date, INTERVAL 1 DAY)
3555
-- Filter out rows from yesterday that have now fallen outside the 28-day window.
36-
AND udf.shift_28_bits_one_day(days_seen_bits) > 0
56+
AND `moz-fx-data-shared-prod`.udf.shift_28_bits_one_day(days_seen_bits) > 0
3757
)
3858
SELECT
3959
@submission_date AS submission_date,
4060
IF(_current.usage_profile_id IS NOT NULL, _current, _previous).* REPLACE (
41-
udf.combine_adjacent_days_28_bits(
61+
`moz-fx-data-shared-prod`.udf.combine_adjacent_days_28_bits(
4262
_previous.days_seen_bits,
4363
_current.days_seen_bits
4464
) AS days_seen_bits,
45-
udf.combine_adjacent_days_28_bits(
65+
`moz-fx-data-shared-prod`.udf.combine_adjacent_days_28_bits(
4666
_previous.days_active_bits,
4767
_current.days_active_bits
4868
) AS days_active_bits,
49-
udf.combine_adjacent_days_28_bits(
69+
`moz-fx-data-shared-prod`.udf.combine_adjacent_days_28_bits(
5070
_previous.days_created_profile_bits,
5171
_current.days_created_profile_bits
5272
) AS days_created_profile_bits
53-
)
73+
),
5474
FROM
5575
_current
5676
FULL JOIN

0 commit comments

Comments
 (0)