feat: Add transformer role and some chore actions (#4)

jakubigla · web-flow · commit aa5a8103ac03 · 2023-02-06T15:24:31.000+01:00
diff --git a/README.md b/README.md
@@ -17,8 +17,9 @@ Terraform module for Snowflake database management.
 * Creates Snowflake database
 * Can create custom Snowflake roles with role-to-role, role-to-user assignments
 * Can create a set of default roles to simplify access management:
-    * `READONLY` - granted 
-    * `ADMIN` - Full access, including database options like `data_retention_time_in_days`
+    * `READONLY` - granted `USAGE` privilege on the database
+    * `TRANSFORMER` - allows creating schemas and some Snowflake objects in them
+    * `ADMIN` - full access, including database options like `data_retention_time_in_days`
 * Can create number of schemas in the database with their specific access roles
 
 ## USAGE
@@ -73,8 +74,8 @@ module "snowflake_database" {
 | <a name="input_name"></a> [name](#input\_name) | ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'.<br>This is the only ID element not also included as a `tag`.<br>The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. | `string` | `null` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique | `string` | `null` | no |
 | <a name="input_regex_replace_chars"></a> [regex\_replace\_chars](#input\_regex\_replace\_chars) | Terraform regular expression (regex) string.<br>Characters matching the regex will be removed from the ID elements.<br>If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no |
-| <a name="input_roles"></a> [roles](#input\_roles) | Roles created in the database scope | <pre>map(object({<br>    enabled              = optional(bool, true)<br>    comment              = optional(string)<br>    role_ownership_grant = optional(string)<br>    granted_roles        = optional(list(string))<br>    granted_to_roles     = optional(list(string))<br>    granted_to_users     = optional(list(string))<br>    database_grants      = optional(list(string))<br>    schema_grants        = optional(list(string))<br>  }))</pre> | `{}` | no |
-| <a name="input_schemas"></a> [schemas](#input\_schemas) | Schemas to be created in the database | <pre>map(object({<br>    descriptor_name     = optional(string, "snowflake-schema")<br>    comment             = optional(string)<br>    data_retention_days = optional(number, 1)<br>    is_transient        = optional(bool, false)<br>    is_managed          = optional(bool, false)<br>    stages = optional(map(object({<br>      enabled              = optional(bool, true)<br>      descriptor_name      = optional(string, "snowflake-stage")<br>      aws_external_id      = optional(string)<br>      comment              = optional(string)<br>      copy_options         = optional(string)<br>      credentials          = optional(string)<br>      directory            = optional(string)<br>      encryption           = optional(string)<br>      file_format          = optional(string)<br>      snowflake_iam_user   = optional(string)<br>      storage_integration  = optional(string)<br>      url                  = optional(string)<br>      create_default_roles = optional(bool)<br>      roles = optional(map(object({<br>        enabled              = optional(bool, true)<br>        comment              = optional(string)<br>        role_ownership_grant = optional(string)<br>        granted_roles        = optional(list(string))<br>        granted_to_roles     = optional(list(string))<br>        granted_to_users     = optional(list(string))<br>        stage_grants         = optional(list(string))<br>      })), {})<br>    })), {})<br>    create_default_roles = optional(bool)<br>    roles = optional(map(object({<br>      enabled                  = optional(bool, true)<br>      comment                  = optional(string)<br>      role_ownership_grant     = optional(string)<br>      granted_roles            = optional(list(string))<br>      granted_to_roles         = optional(list(string))<br>      granted_to_users         = optional(list(string))<br>      schema_grants            = optional(list(string))<br>      table_grants             = optional(list(string))<br>      external_table_grants    = optional(list(string))<br>      view_grants              = optional(list(string))<br>      materialized_view_grants = optional(list(string))<br>      file_format_grants       = optional(list(string))<br>      function_grants          = optional(list(string))<br>      stage_grants             = optional(list(string))<br>      task_grants              = optional(list(string))<br>      procedure_grants         = optional(list(string))<br>      sequence_grants          = optional(list(string))<br>      stream_grants            = optional(list(string))<br>    })), {})<br>  }))</pre> | `{}` | no |
+| <a name="input_roles"></a> [roles](#input\_roles) | Roles created in the database scope | <pre>map(object({<br>    enabled              = optional(bool, true)<br>    descriptor_name      = optional(string, "snowflake-role")<br>    comment              = optional(string)<br>    role_ownership_grant = optional(string)<br>    granted_roles        = optional(list(string))<br>    granted_to_roles     = optional(list(string))<br>    granted_to_users     = optional(list(string))<br>    database_grants      = optional(list(string))<br>    schema_grants        = optional(list(string))<br>  }))</pre> | `{}` | no |
+| <a name="input_schemas"></a> [schemas](#input\_schemas) | Schemas to be created in the database | <pre>map(object({<br>    enabled              = optional(bool, true)<br>    skip_schema_creation = optional(bool, false)<br>    descriptor_name      = optional(string, "snowflake-schema")<br>    comment              = optional(string)<br>    data_retention_days  = optional(number, 1)<br>    is_transient         = optional(bool, false)<br>    is_managed           = optional(bool, false)<br>    stages = optional(map(object({<br>      enabled              = optional(bool, true)<br>      descriptor_name      = optional(string, "snowflake-stage")<br>      aws_external_id      = optional(string)<br>      comment              = optional(string)<br>      copy_options         = optional(string)<br>      credentials          = optional(string)<br>      directory            = optional(string)<br>      encryption           = optional(string)<br>      file_format          = optional(string)<br>      snowflake_iam_user   = optional(string)<br>      storage_integration  = optional(string)<br>      url                  = optional(string)<br>      create_default_roles = optional(bool)<br>      roles = optional(map(object({<br>        enabled              = optional(bool, true)<br>        descriptor_name      = optional(string, "snowflake-role")<br>        comment              = optional(string)<br>        role_ownership_grant = optional(string)<br>        granted_roles        = optional(list(string))<br>        granted_to_roles     = optional(list(string))<br>        granted_to_users     = optional(list(string))<br>        stage_grants         = optional(list(string))<br>      })), {})<br>    })), {})<br>    create_default_roles = optional(bool)<br>    roles = optional(map(object({<br>      enabled                  = optional(bool, true)<br>      descriptor_name          = optional(string, "snowflake-role")<br>      comment                  = optional(string)<br>      role_ownership_grant     = optional(string)<br>      granted_roles            = optional(list(string))<br>      granted_to_roles         = optional(list(string))<br>      granted_to_users         = optional(list(string))<br>      schema_grants            = optional(list(string))<br>      table_grants             = optional(list(string))<br>      external_table_grants    = optional(list(string))<br>      view_grants              = optional(list(string))<br>      materialized_view_grants = optional(list(string))<br>      file_format_grants       = optional(list(string))<br>      function_grants          = optional(list(string))<br>      stage_grants             = optional(list(string))<br>      task_grants              = optional(list(string))<br>      procedure_grants         = optional(list(string))<br>      sequence_grants          = optional(list(string))<br>      stream_grants            = optional(list(string))<br>    })), {})<br>  }))</pre> | `{}` | no |
 | <a name="input_stage"></a> [stage](#input\_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`).<br>Neither the tag keys nor the tag values will be modified by this module. | `map(string)` | `{}` | no |
 | <a name="input_tenant"></a> [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no |
@@ -87,7 +88,7 @@ module "snowflake_database" {
 | <a name="module_roles_deep_merge"></a> [roles\_deep\_merge](#module\_roles\_deep\_merge) | Invicton-Labs/deepmerge/null | 0.1.5 |
 | <a name="module_snowflake_custom_role"></a> [snowflake\_custom\_role](#module\_snowflake\_custom\_role) | getindata/role/snowflake | 1.0.3 |
 | <a name="module_snowflake_default_role"></a> [snowflake\_default\_role](#module\_snowflake\_default\_role) | getindata/role/snowflake | 1.0.3 |
-| <a name="module_snowflake_schema"></a> [snowflake\_schema](#module\_snowflake\_schema) | getindata/schema/snowflake | 1.1.0 |
+| <a name="module_snowflake_schema"></a> [snowflake\_schema](#module\_snowflake\_schema) | getindata/schema/snowflake | 1.3.0 |
 | <a name="module_this"></a> [this](#module\_this) | cloudposse/label/null | 0.25.0 |
 
 ## Outputs
diff --git a/locals.tf b/locals.tf
@@ -5,28 +5,53 @@ locals {
     lookup(module.database_label.descriptors, var.descriptor_name, module.database_label.id), "/${module.database_label.delimiter}${module.database_label.delimiter}+/", module.database_label.delimiter
   ), module.database_label.delimiter) : null
 
+  enabled              = module.this.enabled
   create_default_roles = module.this.enabled && var.create_default_roles
 
+  #This needs to be the same as an object in roles variable
+  role_template = {
+    enabled              = true
+    descriptor_name      = "snowflake-role"
+    comment              = null
+    role_ownership_grant = "SYSADMIN"
+    granted_roles        = []
+    granted_to_roles     = []
+    granted_to_users     = []
+    database_grants      = []
+    schema_grants        = []
+  }
   default_roles_definition = {
     readonly = {
       database_grants = ["USAGE", "MONITOR"]
+      database_grants = ["USAGE"]
+    }
+    transformer = {
+      database_grants = ["USAGE", "MONITOR", "CREATE SCHEMA"]
+      schema_grants   = ["USAGE", "CREATE TEMPORARY TABLE", "CREATE TAG", "CREATE PIPE", "CREATE PROCEDURE", "CREATE MATERIALIZED VIEW", "CREATE TABLE", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE TASK", "CREATE FUNCTION", "CREATE EXTERNAL TABLE", "CREATE SEQUENCE", "CREATE VIEW", "CREATE STREAM"]
     }
     admin = {
-      database_grants = ["USAGE", "MONITOR", "MODIFY", "OWNERSHIP", "REFERENCE_USAGE", "CREATE SCHEMA"]
-      schema_grants   = ["MONITOR", "CREATE TEMPORARY TABLE", "CREATE TAG", "CREATE PIPE", "CREATE PROCEDURE", "CREATE MATERIALIZED VIEW", "CREATE ROW ACCESS POLICY", "USAGE", "CREATE TABLE", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE TASK", "CREATE FUNCTION", "CREATE EXTERNAL TABLE", "ADD SEARCH OPTIMIZATION", "MODIFY", "OWNERSHIP", "CREATE SEQUENCE", "CREATE MASKING POLICY", "CREATE VIEW", "CREATE STREAM"]
+      database_grants = ["USAGE", "MONITOR", "MODIFY", "REFERENCE_USAGE", "CREATE SCHEMA"]
+      schema_grants   = ["USAGE", "MONITOR", "MODIFY", "CREATE TEMPORARY TABLE", "CREATE TAG", "CREATE PIPE", "CREATE PROCEDURE", "CREATE MATERIALIZED VIEW", "CREATE ROW ACCESS POLICY", "CREATE TABLE", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE TASK", "CREATE FUNCTION", "CREATE EXTERNAL TABLE", "ADD SEARCH OPTIMIZATION", "CREATE SEQUENCE", "CREATE MASKING POLICY", "CREATE VIEW", "CREATE STREAM"]
     }
   }
 
   provided_roles = { for role_name, role in var.roles : role_name => {
     for k, v in role : k => v
     if v != null
   } }
-  roles_definition = module.roles_deep_merge.merged
+
+  roles_definition = {
+    for role_name, role in module.roles_deep_merge.merged : role_name => merge(
+      local.role_template,
+      role
+    )
+  }
 
   default_roles = {
     for role_name, role in local.roles_definition : role_name => role
     if contains(keys(local.default_roles_definition), role_name)
   }
+
   custom_roles = {
     for role_name, role in local.roles_definition : role_name => role
     if !contains(keys(local.default_roles_definition), role_name)
@@ -39,6 +64,8 @@ locals {
     ) : role_name => role
     if role.name != null
   }
+
+  schemas = var.schemas
 }
 
 module "roles_deep_merge" {
diff --git a/main.tf b/main.tf
@@ -27,49 +27,55 @@ module "snowflake_default_role" {
 
   source  = "getindata/role/snowflake"
   version = "1.0.3"
-  context = module.this.context
-  enabled = local.create_default_roles && lookup(each.value, "enabled", true)
+
+  context         = module.this.context
+  enabled         = local.create_default_roles && each.value.enabled
+  descriptor_name = each.value.descriptor_name
 
   name       = each.key
   attributes = [one(snowflake_database.this[*].name)]
 
-  role_ownership_grant = lookup(each.value, "role_ownership_grant", "SYSADMIN")
-  granted_to_users     = lookup(each.value, "granted_to_users", [])
-  granted_to_roles     = lookup(each.value, "granted_to_roles", [])
-  granted_roles        = lookup(each.value, "granted_roles", [])
+  role_ownership_grant = each.value.role_ownership_grant
+  granted_to_users     = each.value.granted_to_users
+  granted_to_roles     = each.value.granted_to_roles
+  granted_roles        = each.value.granted_roles
 }
 
-
 module "snowflake_custom_role" {
   for_each = local.custom_roles
 
   source  = "getindata/role/snowflake"
   version = "1.0.3"
-  context = module.this.context
-  enabled = module.this.enabled && lookup(each.value, "enabled", true)
+
+  context         = local.enabled
+  enabled         = module.this.enabled && each.value.enabled
+  descriptor_name = each.value.descriptor_name
 
   name       = each.key
   attributes = [one(snowflake_database.this[*].name)]
 
-  role_ownership_grant = lookup(each.value, "role_ownership_grant", "SYSADMIN")
-  granted_to_users     = lookup(each.value, "granted_to_users", [])
-  granted_to_roles     = lookup(each.value, "granted_to_roles", [])
-  granted_roles        = lookup(each.value, "granted_roles", [])
+  role_ownership_grant = each.value.role_ownership_grant
+  granted_to_users     = each.value.granted_to_users
+  granted_to_roles     = each.value.granted_to_roles
+  granted_roles        = each.value.granted_roles
 }
 
 module "snowflake_schema" {
-  for_each = var.schemas
+  for_each = local.schemas
 
   source  = "getindata/schema/snowflake"
-  version = "1.1.0"
+  version = "1.3.0"
+
   context = module.this.context
+  enabled = each.value.enabled
 
   name     = each.key
   database = one(snowflake_database.this[*].name)
 
-  data_retention_days = each.value.data_retention_days
-  is_transient        = each.value.is_transient
-  is_managed          = each.value.is_managed
+  skip_schema_creation = each.value.skip_schema_creation
+  data_retention_days  = each.value.data_retention_days
+  is_transient         = each.value.is_transient
+  is_managed           = each.value.is_managed
 
   stages = each.value.stages
 
diff --git a/variables.tf b/variables.tf
@@ -53,6 +53,7 @@ variable "roles" {
   description = "Roles created in the database scope"
   type = map(object({
     enabled              = optional(bool, true)
+    descriptor_name      = optional(string, "snowflake-role")
     comment              = optional(string)
     role_ownership_grant = optional(string)
     granted_roles        = optional(list(string))
@@ -67,11 +68,13 @@ variable "roles" {
 variable "schemas" {
   description = "Schemas to be created in the database"
   type = map(object({
-    descriptor_name     = optional(string, "snowflake-schema")
-    comment             = optional(string)
-    data_retention_days = optional(number, 1)
-    is_transient        = optional(bool, false)
-    is_managed          = optional(bool, false)
+    enabled              = optional(bool, true)
+    skip_schema_creation = optional(bool, false)
+    descriptor_name      = optional(string, "snowflake-schema")
+    comment              = optional(string)
+    data_retention_days  = optional(number, 1)
+    is_transient         = optional(bool, false)
+    is_managed           = optional(bool, false)
     stages = optional(map(object({
       enabled              = optional(bool, true)
       descriptor_name      = optional(string, "snowflake-stage")
@@ -88,6 +91,7 @@ variable "schemas" {
       create_default_roles = optional(bool)
       roles = optional(map(object({
         enabled              = optional(bool, true)
+        descriptor_name      = optional(string, "snowflake-role")
         comment              = optional(string)
         role_ownership_grant = optional(string)
         granted_roles        = optional(list(string))
@@ -99,6 +103,7 @@ variable "schemas" {
     create_default_roles = optional(bool)
     roles = optional(map(object({
       enabled                  = optional(bool, true)
+      descriptor_name          = optional(string, "snowflake-role")
       comment                  = optional(string)
       role_ownership_grant     = optional(string)
       granted_roles            = optional(list(string))
