diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql index 2272dbcd343..a96b667588f 100755 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/event_events_v1/query.sql @@ -53,30 +53,36 @@ FROM base WHERE -- See https://bugzilla.mozilla.org/show_bug.cgi?id=1703362 - NOT ( + ( event_category = 'security' AND event_method = 'unexpectedload' AND mozfun.map.get_key(event_map_values, 'contenttype') = 'TYPE_STYLESHEET' - AND mozfun.norm.truncate_version(app_version, 'major') + AND mozfun.norm.extract_version(app_version, 'major') BETWEEN 84 AND 87 - ) - AND + ) IS NOT TRUE -- See https://bugzilla.mozilla.org/show_bug.cgi?id=1803833 - NOT ( + AND ( event_category = 'normandy' AND event_method = 'validationFailed' AND mozfun.map.get_key(event_map_values, 'reason') = 'invalid-feature' AND mozfun.map.get_key(event_map_values, 'feature') IN ('nimbus-qa-1', 'nimbus-qa-2') - AND mozfun.norm.truncate_version(app_version, 'major') <= 108 - ) - AND + AND mozfun.norm.extract_version(app_version, 'major') <= 108 + ) IS NOT TRUE -- See https://mozilla-hub.atlassian.net/browse/DENG-7513 - NOT ( + AND ( event_category = 'security' AND event_method = 'unexpectedload' AND normalized_channel = 'release' - AND mozfun.norm.truncate_version(app_version, 'major') + AND mozfun.norm.extract_version(app_version, 'major') BETWEEN 133 AND 135 - ) + ) IS NOT TRUE + -- See https://mozilla-hub.atlassian.net/browse/DENG-9732 + AND ( + event_category = "uptake.remotecontent.result" + AND event_method = "uptake" + AND normalized_channel = 'release' + AND mozfun.norm.extract_version(app_version, 'major') >= 143 + AND sample_id != 0 + ) IS NOT TRUE diff --git a/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql b/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql index e3bdb46f1d1..2d9ae040f59 100644 --- a/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql +++ b/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql @@ -8,17 +8,19 @@ WITH {% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%} base_{{ dataset['bq_dataset_family'] }}_{{ events_table }} AS ( SELECT - submission_timestamp, + @submission_date AS submission_date, + TIMESTAMP_TRUNC(submission_timestamp, HOUR) AS window_start, + TIMESTAMP_ADD(TIMESTAMP_TRUNC(submission_timestamp, HOUR), INTERVAL 1 HOUR) AS window_end, event.category AS event_category, event.name AS event_name, event_extra.key AS event_extra_key, normalized_country_code AS country, - "{{ dataset['canonical_app_name'] }}" AS normalized_app_name, client_info.app_channel AS channel, client_info.app_display_version AS version, -- experiments[ARRAY_LENGTH(experiments)] will be set to '*' COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].key, '*') AS experiment, COALESCE(ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch, '*') AS experiment_branch, + COUNT(*) AS total_events, FROM `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ events_table }}` CROSS JOIN @@ -30,22 +32,46 @@ WITH LEFT JOIN -- Add * extra to every event to get total event count UNNEST(event.extra || [STRUCT('*', NULL)]) AS event_extra + WHERE + DATE(submission_timestamp) = @submission_date + {% if dataset['app_name'] == "firefox_desktop" and events_table == "events_v1" %} + -- See https://mozilla-hub.atlassian.net/browse/DENG-9732 + AND ( + event.category = "uptake.remotecontent.result" + AND event.name IN ("uptake_remotesettings", "uptake_normandy") + AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143 + AND sample_id != 0 + ) IS NOT TRUE + {% endif %} + GROUP BY + submission_date, + window_start, + window_end, + event_category, + event_name, + event_extra_key, + country, + channel, + version, + experiment, + experiment_branch ), {% endfor %} {{ dataset['bq_dataset_family'] }}_aggregated AS ( SELECT - @submission_date AS submission_date, - TIMESTAMP_ADD( - TIMESTAMP_TRUNC(submission_timestamp, HOUR), - -- Aggregates event counts over 60-minute intervals - INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE - ) AS window_start, - TIMESTAMP_ADD( - TIMESTAMP_TRUNC(submission_timestamp, HOUR), - INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE - ) AS window_end, - * EXCEPT (submission_timestamp), - COUNT(*) AS total_events, + submission_date, + window_start, + window_end, + event_category, + event_name, + event_extra_key, + country, + "{{ dataset['canonical_app_name'] }}" AS normalized_app_name, + channel, + version, + experiment, + experiment_branch, + SUM(total_events) AS total_events, FROM ( {% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%} @@ -56,8 +82,6 @@ WITH {{ "UNION ALL" if not loop.last }} {% endfor -%} ) - WHERE - DATE(submission_timestamp) = @submission_date GROUP BY submission_date, window_start, diff --git a/sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql b/sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql index 59167333867..48dc910ceae 100644 --- a/sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql +++ b/sql_generators/glean_usage/templates/event_monitoring_live_v1.materialized_view.sql @@ -31,6 +31,17 @@ IF LEFT JOIN -- Add * extra to every event to get total event count UNNEST(event.extra || [STRUCT('*', NULL)]) AS event_extra + {% if dataset == "firefox_desktop" and events_table == "events_v1" %} + WHERE + -- See https://mozilla-hub.atlassian.net/browse/DENG-9732 + ( + normalized_channel = 'release' + AND event.category = 'uptake.remotecontent.result' + AND event.name IN ('uptake_remotesettings', 'uptake_normandy') + AND mozfun.norm.extract_version(client_info.app_display_version, 'major') >= 143 + AND sample_id != 0 + ) IS NOT TRUE + {% endif %} ){{ "," if not loop.last }} {% endfor -%}, combined AS ( @@ -46,15 +57,8 @@ IF SELECT -- used for partitioning, only allows TIMESTAMP columns TIMESTAMP_TRUNC(submission_timestamp, DAY) AS submission_date, - TIMESTAMP_ADD( - TIMESTAMP_TRUNC(submission_timestamp, HOUR), - -- Aggregates event counts over 60-minute intervals - INTERVAL(DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) * 60) MINUTE - ) AS window_start, - TIMESTAMP_ADD( - TIMESTAMP_TRUNC(submission_timestamp, HOUR), - INTERVAL((DIV(EXTRACT(MINUTE FROM submission_timestamp), 60) + 1) * 60) MINUTE - ) AS window_end, + TIMESTAMP_TRUNC(submission_timestamp, HOUR) AS window_start, + TIMESTAMP_ADD(TIMESTAMP_TRUNC(submission_timestamp, HOUR), INTERVAL 1 HOUR) AS window_end, * EXCEPT (submission_timestamp), COUNT(*) AS total_events, FROM diff --git a/sql_generators/glean_usage/templates/events_stream_v1.query.sql b/sql_generators/glean_usage/templates/events_stream_v1.query.sql index 400bd7933c9..d80ed5cfd53 100644 --- a/sql_generators/glean_usage/templates/events_stream_v1.query.sql +++ b/sql_generators/glean_usage/templates/events_stream_v1.query.sql @@ -153,6 +153,14 @@ CROSS JOIN AND event.name = 'enrollment_status' AND app_version_major = 140 ) IS NOT TRUE + -- See https://mozilla-hub.atlassian.net/browse/DENG-9732 + AND ( + normalized_channel = 'release' + AND event.category = 'uptake.remotecontent.result' + AND event.name IN ('uptake_remotesettings', 'uptake_normandy') + AND app_version_major >= 143 + AND sample_id != 0 + ) IS NOT TRUE {% elif app_name == "firefox_desktop_background_update" %} WHERE -- See https://mozilla-hub.atlassian.net/browse/DENG-8432