Bug 1988207 - Look for schema in output_dir in glean_app_ping_views (#8096)

* Bug 1988207 - Look for schema in output_dir in glean_app_ping_views

* add default sql_dir

Author: BenWu

bigquery_etl/schema/__init__.py

@@ -363,16 +363,26 @@ def recurse_fields(
 
                 if node_name in source_schema_nodes:  # field exists in app schema
                     # field matches, can query as-is
-                    if node == source_schema_nodes[node_name] and (
-                        # don't need to unnest scalar
-                        dtype != "RECORD"
-                        or not unnest_structs
-                        # reached max record depth to unnest
-                        or len(node_path) > max_unnest_depth > 0
-                        # field not in unnest allowlist
-                        or (
-                            unnest_allowlist is not None
-                            and node_path[0] not in unnest_allowlist
+                    if (
+                        dtype == source_schema_nodes[node["name"]]["type"]
+                        and (
+                            # fields must match for records, otherwise only type needs to match
+                            # note that this will be false if field descriptions don't match
+                            dtype != "RECORD"
+                            or node["fields"]
+                            == source_schema_nodes[node_name]["fields"]
+                        )
+                        and (
+                            # don't need to unnest scalar
+                            dtype != "RECORD"
+                            or not unnest_structs
+                            # reached max record depth to unnest
+                            or len(node_path) > max_unnest_depth > 0
+                            # field not in unnest allowlist
+                            or (
+                                unnest_allowlist is not None
+                                and node_path[0] not in unnest_allowlist
+                            )
                         )
                     ):
                         if (

sql_generators/glean_usage/glean_app_ping_views.py

@@ -116,13 +116,14 @@ def _process_ping(ping_name):
                 if channel_dataset_view in ignored_pings:
                     continue
 
-                sql_dir = Path(ConfigLoader.get("default", "sql_dir", fallback="sql"))
+                # look for schema in output_dir because bqetl generate all runs stable_views first
+                sql_dir = (
+                    output_dir
+                    or Path(ConfigLoader.get("default", "sql_dir", fallback="sql"))
+                    / project_id
+                )
                 existing_schema_path = (
-                    sql_dir
-                    / "moz-fx-data-shared-prod"
-                    / channel_dataset
-                    / view_name
-                    / SCHEMA_FILE
+                    sql_dir / channel_dataset / view_name / SCHEMA_FILE
                 )
 
                 schema = None

tests/schema/test_schema.py

@@ -689,3 +689,62 @@ def test_generate_select_expression_unnest_allowlist():
         unnest_structs=True, unnest_allowlist=["record2"]
     )
     assert reformat(unnest_expr) == reformat(expected_expr)
+
+
+def test_generate_compatible_select_expression_description():
+    source_schema = {
+        "fields": [
+            {"name": "scalar", "type": "INTEGER", "description": "abc"},
+            {
+                "name": "record",
+                "type": "RECORD",
+                "fields": [
+                    {"name": "nested", "type": "DATE"},
+                ],
+            },
+            {
+                "name": "record2",
+                "type": "RECORD",
+                "fields": [
+                    {"name": "nested", "type": "DATE"},
+                ],
+                "description": "abc",
+            },
+        ]
+    }
+    target_schema = {
+        "fields": [
+            {"name": "scalar", "type": "INTEGER"},
+            {
+                "name": "record",
+                "type": "RECORD",
+                "fields": [
+                    {"name": "nested", "type": "DATE", "description": "abc"},
+                ],
+            },
+            {
+                "name": "record2",
+                "type": "RECORD",
+                "fields": [
+                    {"name": "nested", "type": "DATE"},
+                ],
+            },
+        ]
+    }
+
+    # RECORDs will be unnested if nested descriptions don't match
+    expected_expr = """
+    scalar,
+    STRUCT(
+        record.nested
+    ) AS `record`,
+    record2
+    """
+
+    source = Schema.from_json(source_schema)
+    target = Schema.from_json(target_schema)
+
+    select_expr = source.generate_compatible_select_expression(
+        target, unnest_structs=False
+    )
+    assert reformat(select_expr) == reformat(expected_expr)