From 1e395b9c117508da2678c536b087874d2405d93b Mon Sep 17 00:00:00 2001 From: Liam McFall Date: Mon, 22 Sep 2025 15:19:38 -0400 Subject: [PATCH 1/4] feat: build win10 user targeting for braze --- .../fxa_win10_users_v1/metadata.yaml | 22 ++++++++ .../fxa_win10_users_v1/query.sql | 56 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml new file mode 100644 index 00000000000..e8d8caa7f08 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fxa Win10 Users +description: |- + This query identifies Firefox Accounts users on Windows 10 who have been inactive for exactly 14 days and + prepares their records for Braze by assigning an external ID and email. +owners: +- lmcfall@mozilla.com +labels: + incremental: false + schedule: daily + owner: lmcfall +scheduling: + dag_name: bqetl_braze + date_partition_parameter: submission_date +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:accounts-confidential diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql new file mode 100644 index 00000000000..64ed16edba0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql @@ -0,0 +1,56 @@ +WITH win10_users AS ( + SELECT + TO_HEX(SHA256(metrics.string.client_association_uid)) AS fxa_id_sha256, + client_info.os AS os, + client_info.os_version AS os_version + FROM + `moz-fx-data-shared-prod.firefox_desktop.fx_accounts` + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.os = 'Windows' + AND client_info.os_version = '10.0' + GROUP BY + 1, + 2, + 3 +), +last_seen_14_days AS ( + SELECT + * + FROM + `moz-fx-data-shared-prod.accounts_backend_derived.users_services_last_seen_v1` + WHERE + submission_date = @submission_date + -- bit pattern 10000000000000, last seen 14 days from submission date + AND days_seen_bits = 8192 +), +inactive_win10_users AS ( + SELECT + last_seen.user_id_sha256, + win10.os, + win10.os_version, + last_seen.days_seen_bits + FROM + last_seen_14_days AS last_seen + LEFT JOIN + win10_users AS win10 + ON last_seen.user_id_sha256 = win10.fxa_id_sha256 + -- filter out users that don't have an FX account + WHERE + win10.fxa_id_sha256 IS NOT NULL +) +SELECT + inactive.submission_date, + -- if user is in our braze users table use their external_id, otherwise generate a braze external_id + IFNULL(braze_users.external_id, GENERATE_UUID()) AS external_id, + -- if user is in our braze users table use their email, otherwise use the email associated with their fxa_id + IFNULL(braze_users.email, fxa_emails.normalizedEmail) AS email, + inactive.fxa_id_sha256 +FROM + inactive_win10_users AS inactive +LEFT JOIN + `moz-fx-data-shared-prod.braze_derived.users_v1` AS braze_users + ON inactive.user_id_sha256 = braze_users.fxa_id_sha256 +LEFT JOIN + `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` AS fxa_emails + ON inactive.fxa_id_sha256 = fxa_emails.uid From 00ef716c2da84aa792971212d89212e4dd971dbd Mon Sep 17 00:00:00 2001 From: Liam McFall Date: Thu, 25 Sep 2025 16:33:09 -0400 Subject: [PATCH 2/4] reworking win10 users to handle duplication and create braze sync table --- .../fxa_win10_users_v1/metadata.yaml | 10 ---- .../fxa_win10_users_v1/query.sql | 52 ++++++++++++------- .../win10_users_sync_v1/metadata.yaml | 11 ++++ .../win10_users_sync_v1/query.sql | 17 ++++++ 4 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml index e8d8caa7f08..ba03de8653c 100644 --- a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml @@ -10,13 +10,3 @@ labels: owner: lmcfall scheduling: dag_name: bqetl_braze - date_partition_parameter: submission_date -bigquery: - time_partitioning: - type: day - field: submission_date - require_partition_filter: true -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:accounts-confidential diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql index 64ed16edba0..e6ded1e8174 100644 --- a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql @@ -1,35 +1,50 @@ WITH win10_users AS ( - SELECT + SELECT DISTINCT TO_HEX(SHA256(metrics.string.client_association_uid)) AS fxa_id_sha256, - client_info.os AS os, - client_info.os_version AS os_version + FIRST_VALUE(client_info.os) OVER ( + PARTITION BY + TO_HEX(SHA256(metrics.string.client_association_uid)) + ORDER BY + submission_timestamp DESC + ) AS os, + FIRST_VALUE(client_info.os_version) OVER ( + PARTITION BY + TO_HEX(SHA256(metrics.string.client_association_uid)) + ORDER BY + submission_timestamp DESC + ) AS os_version, + FIRST_VALUE(client_info.locale) OVER ( + PARTITION BY + TO_HEX(SHA256(metrics.string.client_association_uid)) + ORDER BY + submission_timestamp DESC + ) AS locale FROM `moz-fx-data-shared-prod.firefox_desktop.fx_accounts` WHERE DATE(submission_timestamp) = @submission_date AND client_info.os = 'Windows' AND client_info.os_version = '10.0' - GROUP BY - 1, - 2, - 3 ), last_seen_14_days AS ( - SELECT - * + SELECT DISTINCT + user_id_sha256 FROM `moz-fx-data-shared-prod.accounts_backend_derived.users_services_last_seen_v1` WHERE submission_date = @submission_date - -- bit pattern 10000000000000, last seen 14 days from submission date - AND days_seen_bits = 8192 + -- bit pattern 100000000000000, last seen 14 days from submission date + AND days_seen_bits >= 16384 + -- bit pattern 1000000000000000000000, last seen 21 days from submission date + -- Only useful for the initial pull of these users, they will already be in the table for subsquent runs + AND days_seen_bits < 2097152 ), inactive_win10_users AS ( SELECT last_seen.user_id_sha256, win10.os, win10.os_version, - last_seen.days_seen_bits + win10.locale FROM last_seen_14_days AS last_seen LEFT JOIN @@ -40,17 +55,18 @@ inactive_win10_users AS ( win10.fxa_id_sha256 IS NOT NULL ) SELECT - inactive.submission_date, - -- if user is in our braze users table use their external_id, otherwise generate a braze external_id - IFNULL(braze_users.external_id, GENERATE_UUID()) AS external_id, + braze_users.external_id AS external_id, -- if user is in our braze users table use their email, otherwise use the email associated with their fxa_id IFNULL(braze_users.email, fxa_emails.normalizedEmail) AS email, - inactive.fxa_id_sha256 + inactive.user_id_sha256, + inactive.locale FROM inactive_win10_users AS inactive LEFT JOIN `moz-fx-data-shared-prod.braze_derived.users_v1` AS braze_users ON inactive.user_id_sha256 = braze_users.fxa_id_sha256 LEFT JOIN - `moz-fx-data-shared-prod.accounts_db_external.fxa_emails_v1` AS fxa_emails - ON inactive.fxa_id_sha256 = fxa_emails.uid + `moz-fx-data-shared-prod.accounts_backend_external.emails_v1` AS fxa_emails + ON inactive.user_id_sha256 = TO_HEX(SHA256(fxa_emails.uid)) + -- some users have multiple email addresses in this table, only use primary + AND fxa_emails.isPrimary = TRUE diff --git a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml new file mode 100644 index 00000000000..457f6494792 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml @@ -0,0 +1,11 @@ +friendly_name: Win10 Users Sync +description: |- + This table will sync inactive Windows 10 Firefox users to Braze to be contacted + in the Win10 Inactive User Campaign. +owners: +- lmcfall@mozilla.com +labels: + incremental: true + owner1: lmcfall +scheduling: + dag_name: bqetl_braze diff --git a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql new file mode 100644 index 00000000000..99e5cd65fdf --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql @@ -0,0 +1,17 @@ +SELECT + CURRENT_TIMESTAMP() AS updated_at, + IFNULL(external_id, TO_HEX(SHA256(GENERATE_UUID()))) AS external_id, + TO_JSON( + STRUCT( + email AS email, + "subscribed" AS email_subscribe, + ARRAY_AGG( + STRUCT("TODO" AS subscription_group_id, "subscribed" AS subscription_state) + ) AS subscription_groups, + locale AS locale, + user_id_sha256 AS fxa_id_sha256 + ) + ) AS payload, + user_id_sha256 AS fxa_id_sha256 +FROM + `moz-fx-data-shared-prod.braze_derived.fxa_win10_users_v1` From 38e58c4640e11ce63c043a92559bc3a5422aa880 Mon Sep 17 00:00:00 2001 From: Liam McFall Date: Tue, 30 Sep 2025 17:54:53 -0400 Subject: [PATCH 3/4] refactor braze derived and external tables for windows 10 sync --- .../metadata.yaml | 2 +- .../query.sql | 13 +++++++--- .../metadata.yaml | 18 +++++++++++++ .../fxa_win10_users_historical_v1/query.sql | 25 +++++++++++++++++++ .../win10_users_sync_v1/metadata.yaml | 2 +- .../win10_users_sync_v1/query.sql | 16 ++++++------ 6 files changed, 63 insertions(+), 13 deletions(-) rename sql/moz-fx-data-shared-prod/braze_derived/{fxa_win10_users_v1 => fxa_win10_users_daily_v1}/metadata.yaml (89%) rename sql/moz-fx-data-shared-prod/braze_derived/{fxa_win10_users_v1 => fxa_win10_users_daily_v1}/query.sql (88%) create mode 100644 sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/metadata.yaml similarity index 89% rename from sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml rename to sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/metadata.yaml index ba03de8653c..b82a9a95367 100644 --- a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/metadata.yaml @@ -1,4 +1,4 @@ -friendly_name: Fxa Win10 Users +friendly_name: Fxa Win10 Users Daily description: |- This query identifies Firefox Accounts users on Windows 10 who have been inactive for exactly 14 days and prepares their records for Braze by assigning an external ID and email. diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/query.sql similarity index 88% rename from sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql rename to sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/query.sql index e6ded1e8174..ead436e4c75 100644 --- a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_daily_v1/query.sql @@ -1,6 +1,12 @@ WITH win10_users AS ( SELECT DISTINCT TO_HEX(SHA256(metrics.string.client_association_uid)) AS fxa_id_sha256, + FIRST_VALUE(DATE(submission_timestamp)) OVER ( + PARTITION BY + TO_HEX(SHA256(metrics.string.client_association_uid)) + ORDER BY + submission_timestamp DESC + ) AS submission_date, FIRST_VALUE(client_info.os) OVER ( PARTITION BY TO_HEX(SHA256(metrics.string.client_association_uid)) @@ -34,13 +40,11 @@ last_seen_14_days AS ( WHERE submission_date = @submission_date -- bit pattern 100000000000000, last seen 14 days from submission date - AND days_seen_bits >= 16384 - -- bit pattern 1000000000000000000000, last seen 21 days from submission date - -- Only useful for the initial pull of these users, they will already be in the table for subsquent runs - AND days_seen_bits < 2097152 + AND days_seen_bits = 16384 ), inactive_win10_users AS ( SELECT + win10.submission_date, last_seen.user_id_sha256, win10.os, win10.os_version, @@ -55,6 +59,7 @@ inactive_win10_users AS ( win10.fxa_id_sha256 IS NOT NULL ) SELECT + inactive.submission_date, braze_users.external_id AS external_id, -- if user is in our braze users table use their email, otherwise use the email associated with their fxa_id IFNULL(braze_users.email, fxa_emails.normalizedEmail) AS email, diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/metadata.yaml new file mode 100644 index 00000000000..379c34b6335 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/metadata.yaml @@ -0,0 +1,18 @@ +friendly_name: Fxa Win10 Users Historical +description: |- + This query compiles the full list of Win10 users who have reached 14 days of inactivity. + +owners: +- lmcfall@mozilla.com +labels: + incremental: true + schedule: daily + owner: lmcfall +scheduling: + dag_name: bqetl_braze +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: null diff --git a/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/query.sql new file mode 100644 index 00000000000..714a6e12b35 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/braze_derived/fxa_win10_users_historical_v1/query.sql @@ -0,0 +1,25 @@ +WITH current_list AS ( + SELECT + fxa_id_sha256 + FROM + `moz-fx-data-shared-prod.braze_derived.fxa_win10_users_historical_v1` +) +SELECT + daily.submission_date, + IFNULL(daily.external_id, TO_HEX(SHA256(GENERATE_UUID()))) AS external_id, + daily.email, + daily.locale, + daily.user_id_sha256 AS fxa_id_sha256 +FROM + `moz-fx-data-shared-prod.braze_derived.fxa_win10_users_daily_v1` AS daily +LEFT JOIN + current_list AS historical + ON historical.fxa_id_sha256 = daily.user_id_sha256 +WHERE + submission_date = @submission_date + AND daily.email IS NOT NULL + -- FILTER OUT USERS ALREADY IN THE LIST + AND historical.fxa_id_sha256 IS NULL + -- ONLY FOR EMAIL WARMING IN en-US LOCALE + -- REMOVE THIS FILTER BEFORE 10/14/2025 WHEN FULL CAMPAIGN STARTS + AND daily.locale = 'en-US' diff --git a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml index 457f6494792..9f5f37027f0 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml @@ -5,7 +5,7 @@ description: |- owners: - lmcfall@mozilla.com labels: - incremental: true + incremental: false owner1: lmcfall scheduling: dag_name: bqetl_braze diff --git a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql index 99e5cd65fdf..0ec42fa3765 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/query.sql @@ -1,17 +1,19 @@ SELECT CURRENT_TIMESTAMP() AS updated_at, - IFNULL(external_id, TO_HEX(SHA256(GENERATE_UUID()))) AS external_id, + external_id, TO_JSON( STRUCT( email AS email, "subscribed" AS email_subscribe, - ARRAY_AGG( - STRUCT("TODO" AS subscription_group_id, "subscribed" AS subscription_state) + STRUCT( + "718eea53-371c-4cc6-9fdc-1260b1311bd8" AS subscription_group_id, + "subscribed" AS subscription_state ) AS subscription_groups, locale AS locale, - user_id_sha256 AS fxa_id_sha256 + fxa_id_sha256 ) - ) AS payload, - user_id_sha256 AS fxa_id_sha256 + ) AS payload FROM - `moz-fx-data-shared-prod.braze_derived.fxa_win10_users_v1` + `moz-fx-data-shared-prod.braze_derived.fxa_win10_users_historical_v1` +WHERE + submission_date = @submission_date From f0c4d23c26f01d5d814aa811470982bea362efe6 Mon Sep 17 00:00:00 2001 From: Liam McFall Date: Thu, 2 Oct 2025 12:37:48 -0400 Subject: [PATCH 4/4] change incremental strategy to true --- .../braze_external/win10_users_sync_v1/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml index 9f5f37027f0..457f6494792 100644 --- a/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/braze_external/win10_users_sync_v1/metadata.yaml @@ -5,7 +5,7 @@ description: |- owners: - lmcfall@mozilla.com labels: - incremental: false + incremental: true owner1: lmcfall scheduling: dag_name: bqetl_braze