persist view column comments (#893) (#932)

* persist view column comments

* format: whitespace

* extracted get_matched_column macro

* move parenthesis to the calling macro

* changelog

* fix: remove matching column in different case

* fix: remove get_matched_column  macro - not much logic left there.

* escape column comments and add functional test

* Update Features-20230817-130731.yaml

* remove unneeded f string

* add test fixture to view test

* fix fixtures for TestPersistDocsDeltaView

* fix fixtures for TestPersistDocsDeltaView

* formatting

* fix tests

---------

Co-authored-by: Juri Krainjukov <[email protected]>
(cherry picked from commit 1672efd)

Co-authored-by: colin-rogers-dbt <[email protected]>

Author: github-actions[bot]

.changes/unreleased/Features-20230817-130731.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Persist Column level comments when creating views
+time: 2023-08-17T13:07:31.6812862Z
+custom:
+  Author: jurasan
+  Issue: 372

dbt/include/spark/macros/adapters.sql

@@ -223,9 +223,30 @@
   {% endfor %}
 {% endmacro %}
 
+{% macro get_column_comment_sql(column_name, column_dict) -%}
+  {% if column_name in column_dict and column_dict[column_name]["description"] -%}
+    {% set escaped_description = column_dict[column_name]["description"] | replace("'", "\\'") %}
+    {% set column_comment_clause = "comment '" ~ escaped_description ~ "'" %}
+  {%- endif -%}
+  {{ adapter.quote(column_name) }} {{ column_comment_clause }}
+{% endmacro %}
+
+{% macro get_persist_docs_column_list(model_columns, query_columns) %}
+  {% for column_name in query_columns %}
+    {{ get_column_comment_sql(column_name, model_columns) }}
+    {{- ", " if not loop.last else "" }}
+  {% endfor %}
+{% endmacro %}
 
 {% macro spark__create_view_as(relation, sql) -%}
   create or replace view {{ relation }}
+  {% if config.persist_column_docs() -%}
+    {% set model_columns = model.columns %}
+    {% set query_columns = get_columns_in_query(sql) %}
+    (
+    {{ get_persist_docs_column_list(model_columns, query_columns) }}
+    )
+  {% endif %}
   {{ comment_clause() }}
   {%- set contract_config = config.get('contract') -%}
   {%- if contract_config.enforced -%}

tests/functional/adapter/persist_docs/fixtures.py

@@ -21,11 +21,39 @@
 select 1 as id, 'Joe' as name
 """
 
+_MODELS__VIEW_DELTA_MODEL = """
+{{ config(materialized='view') }}
+select id, count(*) as count from {{ ref('table_delta_model') }} group by id
+"""
+
 _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN = """
 {{ config(materialized='table', file_format='delta') }}
 select 1 as id, 'Joe' as different_name
 """
+_VIEW_PROPERTIES_MODELS = """
+version: 2
 
+models:
+  - name: view_delta_model
+    description: |
+      View model description "with double quotes"
+      and with 'single  quotes' as welll as other;
+      '''abc123'''
+      reserved -- characters
+      --
+      /* comment */
+      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+    columns:
+      - name: id
+        description: |
+          id Column description "with double quotes"
+          and with 'single  quotes' as welll as other;
+          '''abc123'''
+          reserved -- characters
+          --
+          /* comment */
+          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+"""
 _PROPERTIES__MODELS = """
 version: 2
 

tests/functional/adapter/persist_docs/test_persist_docs.py

@@ -10,6 +10,8 @@
     _PROPERTIES__MODELS,
     _PROPERTIES__SEEDS,
     _SEEDS__BASIC,
+    _MODELS__VIEW_DELTA_MODEL,
+    _VIEW_PROPERTIES_MODELS,
 )
 
 
@@ -76,6 +78,48 @@ def test_delta_comments(self, project):
                     assert result[2].startswith("Some stuff here and then a call to")
 
 
+@pytest.mark.skip_profile("apache_spark", "spark_session")
+class TestPersistDocsDeltaView:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL,
+            "view_delta_model.sql": _MODELS__VIEW_DELTA_MODEL,
+            "schema.yml": _VIEW_PROPERTIES_MODELS,
+        }
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "models": {
+                "test": {
+                    "+persist_docs": {
+                        "relation": True,
+                        "columns": True,
+                    },
+                }
+            },
+        }
+
+    def test_delta_comments(self, project):
+        run_dbt(["run"])
+
+        results = project.run_sql(
+            "describe extended {schema}.{table}".format(
+                schema=project.test_schema, table="view_delta_model"
+            ),
+            fetch="all",
+        )
+
+        for result in results:
+            if result[0] == "Comment":
+                assert result[1].startswith("View model description")
+            if result[0] == "id":
+                assert result[2].startswith("id Column description")
+            if result[0] == "count":
+                assert result[2] is None
+
+
 @pytest.mark.skip_profile("apache_spark", "spark_session")
 class TestPersistDocsMissingColumn:
     @pytest.fixture(scope="class")