Skip to content

Commit d1f58cb

Browse files
BenWuBenWu
committed
[DENG-9732] Filter out remotecontent events from event etl
1 parent 96cf86c commit d1f58cb

File tree

4 files changed

+66
-27
lines changed

4 files changed

+66
-27
lines changed

‎sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,30 +53,34 @@ FROM
5353
base
5454
WHERE
5555
-- See https://bugzilla.mozilla.org/show_bug.cgi?id=1703362
56-
NOT (
56+
(
5757
event_category = 'security'
5858
AND event_method = 'unexpectedload'
5959
AND mozfun.map.get_key(event_map_values, 'contenttype') = 'TYPE_STYLESHEET'
60-
AND mozfun.norm.truncate_version(app_version, 'major')
60+
AND mozfun.norm.extract_version(app_version, 'major')
6161
BETWEEN 84
6262
AND 87
63-
)
64-
AND
63+
) IS FALSE
6564
-- See https://bugzilla.mozilla.org/show_bug.cgi?id=1803833
66-
NOT (
65+
AND (
6766
event_category = 'normandy'
6867
AND event_method = 'validationFailed'
6968
AND mozfun.map.get_key(event_map_values, 'reason') = 'invalid-feature'
7069
AND mozfun.map.get_key(event_map_values, 'feature') IN ('nimbus-qa-1', 'nimbus-qa-2')
71-
AND mozfun.norm.truncate_version(app_version, 'major') <= 108
72-
)
73-
AND
70+
AND mozfun.norm.extract_version(app_version, 'major') <= 108
71+
) IS FALSE
7472
-- See https://mozilla-hub.atlassian.net/browse/DENG-7513
75-
NOT (
73+
AND (
7674
event_category = 'security'
7775
AND event_method = 'unexpectedload'
7876
AND normalized_channel = 'release'
79-
AND mozfun.norm.truncate_version(app_version, 'major')
77+
AND mozfun.norm.extract_version(app_version, 'major')
8078
BETWEEN 133
8179
AND 135
82-
)
80+
) IS FALSE
81+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
82+
AND (
83+
event_category = "uptake.remotecontent.result"
84+
AND event_method = "uptake"
85+
AND mozfun.norm.extract_version(app_version, 'major') >= 143
86+
) IS FALSE

‎sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,26 @@ WITH
88
{% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%}
99
base_{{ dataset['bq_dataset_family'] }}_{{ events_table }} AS (
1010
SELECT
11-
submission_timestamp,
11+
@submission_date AS submission_date,
12+
TIMESTAMP_ADD(
13+
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
14+
-- Aggregates event counts over 60-minute intervals
15+
INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE
16+
) AS window_start,
17+
TIMESTAMP_ADD(
18+
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
19+
INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE
20+
) AS window_end,
1221
event.category AS event_category,
1322
event.name AS event_name,
1423
event_extra.key AS event_extra_key,
1524
normalized_country_code AS country,
16-
"{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
1725
client_info.app_channel AS channel,
1826
client_info.app_display_version AS version,
1927
-- experiments[ARRAY_LENGTH(experiments)] will be set to '*'
2028
COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].key, '*') AS experiment,
2129
COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch, '*') AS experiment_branch,
30+
COUNT(*) AS total_events,
2231
FROM
2332
`{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ events_table }}`
2433
CROSS JOIN
@@ -30,22 +39,35 @@ WITH
3039
LEFT JOIN
3140
-- Add * extra to every event to get total event count
3241
UNNEST(event.extra || [STRUCT<key STRING, value STRING>('*', NULL)]) AS event_extra
42+
WHERE
43+
DATE(submission_timestamp) = @submission_date
44+
{% if dataset['app_name'] == "firefox_desktop" and events_table == "events_v1" %}
45+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
46+
AND (
47+
event.category = "uptake.remotecontent.result"
48+
AND event.name IN ("uptake_remotesettings", "uptake_normandy")
49+
AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143
50+
) IS FALSE
51+
{% endif %}
52+
GROUP BY
53+
submission_date,
54+
window_start,
55+
window_end,
56+
event_category,
57+
event_name,
58+
event_extra_key,
59+
country,
60+
normalized_app_name,
61+
channel,
62+
version,
63+
experiment,
64+
experiment_branch
3365
),
3466
{% endfor %}
3567
{{ dataset['bq_dataset_family'] }}_aggregated AS (
3668
SELECT
37-
@submission_date AS submission_date,
38-
TIMESTAMP_ADD(
39-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
40-
-- Aggregates event counts over 60-minute intervals
41-
INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE
42-
) AS window_start,
43-
TIMESTAMP_ADD(
44-
TIMESTAMP_TRUNC(submission_timestamp, HOUR),
45-
INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE
46-
) AS window_end,
47-
* EXCEPT (submission_timestamp),
48-
COUNT(*) AS total_events,
69+
"{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
70+
* REPLACE (SUM(total_events) AS total_events),
4971
FROM
5072
(
5173
{% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%}
@@ -56,8 +78,6 @@ WITH
5678
{{ "UNION ALL" if not loop.last }}
5779
{% endfor -%}
5880
)
59-
WHERE
60-
DATE(submission_timestamp) = @submission_date
6181
GROUP BY
6282
submission_date,
6383
window_start,

‎sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,15 @@ IF
3131
LEFT JOIN
3232
-- Add * extra to every event to get total event count
3333
UNNEST(event.extra || [STRUCT<key STRING, value STRING>('*', NULL)]) AS event_extra
34+
{% if dataset == "firefox_desktop" and events_table == "events_v1" %}
35+
WHERE
36+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
37+
AND (
38+
event.category = "uptake.remotecontent.result"
39+
AND event.name IN ("uptake_remotesettings", "uptake_normandy")
40+
AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143
41+
) IS FALSE
42+
{% endif %}
3443
){{ "," if not loop.last }}
3544
{% endfor -%},
3645
combined AS (

‎sql_generators/glean_usage/templates/events_stream_v1.query.sql‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,12 @@ CROSS JOIN
153153
AND event.name = 'enrollment_status'
154154
AND app_version_major = 140
155155
) IS NOT TRUE
156+
-- See https://mozilla-hub.atlassian.net/browse/DENG-9732
157+
AND (
158+
event.category = "uptake.remotecontent.result"
159+
AND event.name IN ("uptake_remotesettings", "uptake_normandy")
160+
AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143
161+
) IS NOT TRUE
156162
{% elif app_name == "firefox_desktop_background_update" %}
157163
WHERE
158164
-- See https://mozilla-hub.atlassian.net/browse/DENG-8432

0 commit comments

Comments
 (0)