Skip to content

Commit 45cec43

Browse files
BenWuBenWu
authored
[DENG-9732] Filter out remotecontent events from event etl (#8140)
* [DENG-9732] Filter out remotecontent events from event etl * oops * Clean up + only filter release channel * Include sample id 0
1 parent 1bc33b1 commit 45cec43

File tree

4 files changed

+78
-36
lines changed

4 files changed

+78
-36
lines changed

‎sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,30 +53,36 @@ FROM
5353
base
5454
WHERE
5555
-- See https://bugzilla.mozilla.org/show_bug.cgi?id=1703362
56-
NOT (
56+
(
5757
event_category = 'security'
5858
AND event_method = 'unexpectedload'
5959
AND mozfun.map.get_key(event_map_values, 'contenttype') = 'TYPE_STYLESHEET'
60-
AND mozfun.norm.truncate_version(app_version, 'major')
60+
AND mozfun.norm.extract_version(app_version, 'major')
6161
BETWEEN 84
6262
AND 87
63-
)
64-
AND
63+
) IS NOT TRUE
6564
-- See https://bugzilla.mozilla.org/show_bug.cgi?id=1803833
66-
NOT (
65+
AND (
6766
event_category = 'normandy'
6867
AND event_method = 'validationFailed'
6968
AND mozfun.map.get_key(event_map_values, 'reason') = 'invalid-feature'
7069
AND mozfun.map.get_key(event_map_values, 'feature') IN ('nimbus-qa-1', 'nimbus-qa-2')
71-
AND mozfun.norm.truncate_version(app_version, 'major') <= 108
72-
)
73-
AND
70+
AND mozfun.norm.extract_version(app_version, 'major') <= 108
71+
) IS NOT TRUE
7472
-- See https://mozilla-hub.atlassian.net/browse/DENG-7513
75-
NOT (
73+
AND (
7674
event_category = 'security'
7775
AND event_method = 'unexpectedload'
7876
AND normalized_channel = 'release'
79-
AND mozfun.norm.truncate_version(app_version, 'major')
77+
AND mozfun.norm.extract_version(app_version, 'major')
8078
BETWEEN 133
8179
AND 135
82-
)
80+
) IS NOT TRUE
81+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
82+
AND (
83+
event_category = "uptake.remotecontent.result"
84+
AND event_method = "uptake"
85+
AND normalized_channel = 'release'
86+
AND mozfun.norm.extract_version(app_version, 'major') >= 143
87+
AND sample_id != 0
88+
) IS NOT TRUE

‎sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql‎

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,19 @@ WITH
88
{% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%}
99
base_{{ dataset['bq_dataset_family'] }}_{{ events_table }} AS (
1010
SELECT
11-
submission_timestamp,
11+
@submission_date AS submission_date,
12+
TIMESTAMP_TRUNC(submission_timestamp, HOUR) AS window_start,
13+
TIMESTAMP_ADD(TIMESTAMP_TRUNC(submission_timestamp, HOUR), INTERVAL 1 HOUR) AS window_end,
1214
event.category AS event_category,
1315
event.name AS event_name,
1416
event_extra.key AS event_extra_key,
1517
normalized_country_code AS country,
16-
"{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
1718
client_info.app_channel AS channel,
1819
client_info.app_display_version AS version,
1920
-- experiments[ARRAY_LENGTH(experiments)] will be set to '*'
2021
COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].key, '*') AS experiment,
2122
COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch, '*') AS experiment_branch,
23+
COUNT(*) AS total_events,
2224
FROM
2325
`{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ events_table }}`
2426
CROSS JOIN
@@ -30,22 +32,46 @@ WITH
3032
LEFT JOIN
3133
-- Add * extra to every event to get total event count
3234
UNNEST(event.extra || [STRUCT<key STRING, value STRING>('*', NULL)]) AS event_extra
35+
WHERE
36+
DATE(submission_timestamp) = @submission_date
37+
{% if dataset['app_name'] == "firefox_desktop" and events_table == "events_v1" %}
38+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
39+
AND (
40+
event.category = "uptake.remotecontent.result"
41+
AND event.name IN ("uptake_remotesettings", "uptake_normandy")
42+
AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143
43+
AND sample_id != 0
44+
) IS NOT TRUE
45+
{% endif %}
46+
GROUP BY
47+
submission_date,
48+
window_start,
49+
window_end,
50+
event_category,
51+
event_name,
52+
event_extra_key,
53+
country,
54+
channel,
55+
version,
56+
experiment,
57+
experiment_branch
3358
),
3459
{% endfor %}
3560
{{ dataset['bq_dataset_family'] }}_aggregated AS (
3661
SELECT
37-
@submission_date AS submission_date,
38-
TIMESTAMP_ADD(
39-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
40-
-- Aggregates event counts over 60-minute intervals
41-
INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE
42-
) AS window_start,
43-
TIMESTAMP_ADD(
44-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
45-
INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE
46-
) AS window_end,
47-
* EXCEPT (submission_timestamp),
48-
COUNT(*) AS total_events,
62+
submission_date,
63+
window_start,
64+
window_end,
65+
event_category,
66+
event_name,
67+
event_extra_key,
68+
country,
69+
"{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
70+
channel,
71+
version,
72+
experiment,
73+
experiment_branch,
74+
SUM(total_events) AS total_events,
4975
FROM
5076
(
5177
{% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%}
@@ -56,8 +82,6 @@ WITH
5682
{{ "UNION ALL" if not loop.last }}
5783
{% endfor -%}
5884
)
59-
WHERE
60-
DATE(submission_timestamp) = @submission_date
6185
GROUP BY
6286
submission_date,
6387
window_start,

‎sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,17 @@ IF
3131
LEFT JOIN
3232
-- Add * extra to every event to get total event count
3333
UNNEST(event.extra || [STRUCT<key STRING, value STRING>('*', NULL)]) AS event_extra
34+
{% if dataset == "firefox_desktop" and events_table == "events_v1" %}
35+
WHERE
36+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
37+
(
38+
normalized_channel = 'release'
39+
AND event.category = 'uptake.remotecontent.result'
40+
AND event.name IN ('uptake_remotesettings', 'uptake_normandy')
41+
AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143
42+
AND sample_id != 0
43+
) IS NOT TRUE
44+
{% endif %}
3445
){{ "," if not loop.last }}
3546
{% endfor -%},
3647
combined AS (
@@ -46,15 +57,8 @@ IF
4657
SELECT
4758
-- used for partitioning, only allows TIMESTAMP columns
4859
TIMESTAMP_TRUNC(submission_timestamp, DAY) AS submission_date,
49-
TIMESTAMP_ADD(
50-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
51-
-- Aggregates event counts over 60-minute intervals
52-
INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE
53-
) AS window_start,
54-
TIMESTAMP_ADD(
55-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
56-
INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE
57-
) AS window_end,
60+
TIMESTAMP_TRUNC(submission_timestamp, HOUR) AS window_start,
61+
TIMESTAMP_ADD(TIMESTAMP_TRUNC(submission_timestamp, HOUR), INTERVAL 1 HOUR) AS window_end,
5862
* EXCEPT (submission_timestamp),
5963
COUNT(*) AS total_events,
6064
FROM

‎sql_generators/glean_usage/templates/events_stream_v1.query.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,14 @@ CROSS JOIN
153153
AND event.name = 'enrollment_status'
154154
AND app_version_major = 140
155155
) IS NOT TRUE
156+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
157+
AND (
158+
normalized_channel = 'release'
159+
AND event.category = 'uptake.remotecontent.result'
160+
AND event.name IN ('uptake_remotesettings', 'uptake_normandy')
161+
AND app_version_major >= 143
162+
AND sample_id != 0
163+
) IS NOT TRUE
156164
{% elif app_name == "firefox_desktop_background_update" %}
157165
WHERE
158166
-- See https://mozilla-hub.atlassian.net/browse/DENG-8432

0 commit comments

Comments
 (0)