diff --git a/README.md b/README.md index 2b5f0248..d9860582 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ For the complete list of changes made to schemachange check out the [CHANGELOG]( 1. [Versioned Script Naming](#versioned-script-naming) 1. [Repeatable Script Naming](#repeatable-script-naming) 1. [Always Script Naming](#always-script-naming) + 1. [Always First Script Naming](#always-first-script-naming) 1. [Script Requirements](#script-requirements) 1. [Using Variables in Scripts](#using-variables-in-scripts) 1. [Secrets filtering](#secrets-filtering) @@ -38,7 +39,7 @@ For the complete list of changes made to schemachange check out the [CHANGELOG]( 1. [Okta Authentication](#okta-authentication) 1. [Configuration](#configuration) 1. [YAML Config File](#yaml-config-file) - 1. [Yaml Jinja support](#yaml-jinja-support) + 1. [YAML Jinja support](#yaml-jinja-support) 1. [Command Line Arguments](#command-line-arguments) 1. [Running schemachange](#running-schemachange) 1. [Prerequisites](#prerequisites) @@ -65,10 +66,12 @@ schemachange expects a directory structure like the following to exist: |-- V1.1.2__second_change.sql |-- R__sp_add_sales.sql |-- R__fn_get_timezone.sql + |-- F__clone_to_qa.sql |-- folder_2 |-- folder_3 |-- V1.1.3__third_change.sql |-- R__fn_sort_ascii.sql + |-- A__permissions.sql ``` The schemachange folder structure is very flexible. The `project_root` folder is specified with the `-f` or `--root-folder` argument. schemachange only pays attention to the filenames, not the paths. Therefore, under the `project_root` folder you are free to arrange the change scripts any way you see fit. You can have as many subfolders (and nested subfolders) as you would like. @@ -129,6 +132,20 @@ e.g. This type of change script is useful for an environment set up after cloning. Always scripts are applied always last. +### Always First Script Naming + +Always First change scripts are executed with every run of schemachange if the configuration option is set to `True`; the default is `False`. This is an addition to the implementation of [Flyway Versioned Migrations](https://flywaydb.org/documentation/concepts/migrations.html#repeatable-migrations). +The script name must following pattern: + +`F__Some_description.sql` + +e.g. + +* F__QA_Clone.sql +* F__STG_Clone.sql + +This type of change script is useful for cloning an environment at the start of the CI/CD process. When a release is created, the first step is to recreate the QA clone off production, so the change scripts are applied to the most current version of the production environment. After QA approves the release, the cloning action is not needed, so the configuration option is set to `False` and the Always First scripts are skipped. Always First scripts are applied first when the configuration option is set to `True`. + ### Script Requirements schemachange is designed to be very lightweight and not impose to many limitations. Each change script can have any number of SQL statements within it and must supply the necessary context, like database and schema names. The context can be supplied by using an explicit `USE ` command or by naming all objects with a three-part name (`..`). schemachange will simply run the contents of each script against the target Snowflake account, in the correct order. @@ -329,6 +346,9 @@ autocommit: false # Display verbose debugging details during execution (the default is False) verbose: false +# Execute Always First scripts which are executed before other script types (the default is False) +always-first: false + # Run schemachange in dry run mode (the default is False) dry-run: false @@ -396,6 +416,7 @@ Parameter | Description --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. -v, --verbose | Display verbose debugging details during execution. The default is 'False'. +-af, --always-first | Enable to execute Always First scripts. These will be executed before all other script types. The default is 'False'. --dry-run | Run schemachange in dry run mode. The default is 'False'. --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' @@ -403,7 +424,7 @@ Parameter | Description #### render This subcommand is used to render a single script to the console. It is intended to support the development and troubleshooting of script that use features from the jinja template engine. -`usage: schemachange render [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [--vars VARS] [-v] script` +`usage: schemachange render [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [--vars VARS] [-v] [-af] script` Parameter | Description --- | --- @@ -412,6 +433,7 @@ Parameter | Description -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across multiple scripts --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"}) -v, --verbose | Display verbose debugging details during execution (the default is False) +-af, --always-first | Enable to execute Always First scripts. These will be executed before all other script types. The default is 'False'. ## Running schemachange diff --git a/schemachange/cli.py b/schemachange/cli.py index 31745f43..c2cfe55b 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -126,7 +126,7 @@ def override_loader(self, loader: jinja2.BaseLoader): # to make unit testing easier self.__environment = jinja2.Environment(loader=loader, **self._env_args) - def render(self, script: str, vars: Dict[str, Any], verbose: bool) -> str: + def render(self, script: str, vars: Dict[str, Any], verbose: bool, always_first: bool) -> str: if not vars: vars = {} # jinja needs posix path @@ -233,13 +233,16 @@ def __init__(self, config): self.oauth_config = config['oauth_config'] self.autocommit = config['autocommit'] self.verbose = config['verbose'] + if self.set_connection_args(): + self.always_first = config['always_first'] self.con = snowflake.connector.connect(**self.conArgs) if not self.autocommit: self.con.autocommit(False) else: print(_err_env_missing) + def __del__(self): if hasattr(self, 'con'): self.con.close() @@ -531,10 +534,11 @@ def deploy_command(config): print(_log_ch_max_version.format(max_published_version_display=max_published_version_display)) # Find all scripts in the root folder (recursively) and sort them correctly - all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose']) + all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose'], config['always_first']) all_script_names = list(all_scripts.keys()) - # Sort scripts such that versioned scripts get applied first and then the repeatable ones. - all_script_names_sorted = sorted_alphanumeric([script for script in all_script_names if script[0] == 'V']) \ + # Sort scripts such that always first scripts get executed first, then versioned scripts, and then the repeatable ones. + all_script_names_sorted = sorted_alphanumeric([script for script in all_script_names if script[0] == 'F']) \ + + sorted_alphanumeric([script for script in all_script_names if script[0] == 'V']) \ + sorted_alphanumeric([script for script in all_script_names if script[0] == 'R']) \ + sorted_alphanumeric([script for script in all_script_names if script[0] == 'A']) @@ -542,6 +546,18 @@ def deploy_command(config): for script_name in all_script_names_sorted: script = all_scripts[script_name] + # Always process with jinja engine + jinja_processor = JinjaTemplateProcessor(project_root=config['root_folder'], + modules_folder=config['modules_folder']) + content = jinja_processor.render(jinja_processor.relpath(script['script_full_path']), config['vars'], \ + config['verbose'], config['always_first']) + + # Execute the Always First script(s) as long as the `always_first` configuration is True. + if script_name[0] == 'F' and config['always_first']: + if config['verbose']: + print(_log_skip_r.format(**script)) + print(_log_apply.format(**script)) + # Apply a versioned-change script only if the version is newer than the most recent change in the database # Apply any other scripts, i.e. repeatable scripts, irrespective of the most recent change in the database if script_name[0] == 'V' and get_alphanum_key(script['script_version']) <= get_alphanum_key(max_published_version): @@ -550,10 +566,6 @@ def deploy_command(config): scripts_skipped += 1 continue - # Always process with jinja engine - jinja_processor = JinjaTemplateProcessor(project_root = config['root_folder'], modules_folder = config['modules_folder']) - content = jinja_processor.render(jinja_processor.relpath(script['script_full_path']), config['vars'], config['verbose']) - # Apply only R scripts where the checksum changed compared to the last execution of snowchange if script_name[0] == 'R': # Compute the checksum for the script @@ -572,6 +584,7 @@ def deploy_command(config): scripts_skipped += 1 continue + # The Always scripts are applied in this step print(_log_apply.format(**script)) if not config['dry_run']: session.apply_change_script(script, content, change_history_table) @@ -594,7 +607,7 @@ def render_command(config, script_path): jinja_processor = JinjaTemplateProcessor(project_root = config['root_folder'], \ modules_folder = config['modules_folder']) content = jinja_processor.render(jinja_processor.relpath(script_path), \ - config['vars'], config['verbose']) + config['vars'], config['verbose'], config['always_first']) checksum = hashlib.sha224(content.encode('utf-8')).hexdigest() print("Checksum %s" % checksum) @@ -635,7 +648,7 @@ def load_schemachange_config(config_file_path: str) -> Dict[str, Any]: def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, \ snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, snowflake_schema, \ change_history_table, vars, create_change_history_table, autocommit, verbose, \ - dry_run, query_tag, oauth_config, **kwargs): + dry_run, query_tag, oauth_config, always_first, **kwargs): # create cli override dictionary # Could refactor to just pass Args as a dictionary? @@ -648,7 +661,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf "change_history_table":change_history_table, "vars":vars, \ "create_change_history_table":create_change_history_table, \ "autocommit":autocommit, "verbose":verbose, "dry_run":dry_run,\ - "query_tag":query_tag, "oauth_config":oauth_config} + "query_tag":query_tag, "oauth_config":oauth_config, "always_first":always_first} cli_inputs = {k:v for (k,v) in cli_inputs.items() if v} # load YAML inputs and convert kebabs to snakes @@ -662,7 +675,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf "snowflake_warehouse":None, "snowflake_database":None, "snowflake_schema":None, \ "change_history_table":None, \ "vars":{}, "create_change_history_table":False, "autocommit":False, "verbose":False, \ - "dry_run":False , "query_tag":None , "oauth_config":None } + "dry_run":False, "query_tag":None, "oauth_config":None, "always_first":False} #insert defualt values for items not populated config.update({ k:v for (k,v) in config_defaults.items() if not k in config.keys()}) @@ -687,7 +700,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf return config -def get_all_scripts_recursively(root_directory, verbose): +def get_all_scripts_recursively(root_directory, verbose, always_first): all_files = dict() all_versions = list() # Walk the entire directory structure recursively @@ -699,11 +712,17 @@ def get_all_scripts_recursively(root_directory, verbose): file_name.strip(), re.IGNORECASE) repeatable_script_name_parts = re.search(r'^([R])__(.+?)\.(?:sql|sql.jinja)$', \ file_name.strip(), re.IGNORECASE) + always_first_script_name_parts = re.search(r'^([F])__(.+?)\.(?:sql|sql.jinja)$', \ + file_name.strip(), re.IGNORECASE) always_script_name_parts = re.search(r'^([A])__(.+?)\.(?:sql|sql.jinja)$', \ file_name.strip(), re.IGNORECASE) # Set script type depending on whether it matches the versioned file naming format - if script_name_parts is not None: + if always_first_script_name_parts is not None and always_first: + script_type = 'F' + if verbose: + print("Found Always First file " + file_full_path) + elif script_name_parts is not None: script_type = 'V' if verbose: print("Found Versioned file " + file_full_path) @@ -732,11 +751,13 @@ def get_all_scripts_recursively(root_directory, verbose): script['script_name'] = script_name script['script_full_path'] = file_full_path script['script_type'] = script_type - script['script_version'] = '' if script_type in ['R', 'A'] else script_name_parts.group(2) + script['script_version'] = '' if script_type in ['R', 'A', 'F'] else script_name_parts.group(2) if script_type == 'R': script['script_description'] = repeatable_script_name_parts.group(2).replace('_', ' ').capitalize() elif script_type == 'A': script['script_description'] = always_script_name_parts.group(2).replace('_', ' ').capitalize() + elif script_type == 'F': + script['script_description'] = always_first_script_name_parts.group(2).replace('_', ' ').capitalize() else: script['script_description'] = script_name_parts.group(3).replace('_', ' ').capitalize() @@ -831,6 +852,7 @@ def main(argv=sys.argv): parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False) parser_deploy.add_argument('--query-tag', type = str, help = 'The string to add to the Snowflake QUERY_TAG session value for each query executed', required = False) parser_deploy.add_argument('--oauth-config', type = json.loads, help = 'Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', required = False) + parser_deploy.add_argument('-af', '--always-first', action='store_true', help = 'Enable to execute Always First scripts. These will be executed before all other script types.', required = False) # TODO test CLI passing of args parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.") @@ -839,6 +861,7 @@ def main(argv=sys.argv): parser_render.add_argument('-m', '--modules-folder', type = str, help = 'The modules folder for jinja macros and templates to be used across multiple scripts', required = False) parser_render.add_argument('--vars', type = json.loads, help = 'Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', required = False) parser_render.add_argument('-v', '--verbose', action='store_true', help = 'Display verbose debugging details during execution (the default is False)', required = False) + parser_render.add_argument('-af', '--always-first', action='store_true', help = 'Enable to execute Always First scripts. These will be executed before all other script types.', required = False) parser_render.add_argument('script', type = str, help = 'The script to render') # The original parameters did not support subcommands. Check if a subcommand has been supplied diff --git a/tests/test_JinjaTemplateProcessor.py b/tests/test_JinjaTemplateProcessor.py index f6bcb1c5..9b659c08 100644 --- a/tests/test_JinjaTemplateProcessor.py +++ b/tests/test_JinjaTemplateProcessor.py @@ -14,7 +14,7 @@ def test_JinjaTemplateProcessor_render_simple_string(): templates = {"test.sql": "some text"} processor.override_loader(DictLoader(templates)) - context = processor.render("test.sql", None, True) + context = processor.render("test.sql", None, True, False) assert context == "some text" @@ -27,7 +27,7 @@ def test_JinjaTemplateProcessor_render_simple_string_expecting_variable_that_doe processor.override_loader(DictLoader(templates)) with pytest.raises(UndefinedError) as e: - context = processor.render("test.sql", None, True) + context = processor.render("test.sql", None, True, False) assert str(e.value) == "'myvar' is undefined" @@ -41,7 +41,7 @@ def test_JinjaTemplateProcessor_render_simple_string_expecting_variable(): vars = json.loads('{"myvar" : "world"}') - context = processor.render("test.sql", vars, True) + context = processor.render("test.sql", vars, True, False) assert context == "Hello world!" @@ -59,6 +59,6 @@ def test_JinjaTemplateProcessor_render_from_subfolder(tmp_path: pathlib.Path): processor = JinjaTemplateProcessor(str(root_folder), None) template_path = processor.relpath(str(script_file)) - context = processor.render(template_path, {}, True) + context = processor.render(template_path, {}, True, False) assert context == "Hello world!" diff --git a/tests/test_get_all_scripts_recursively.py b/tests/test_get_all_scripts_recursively.py index 2d3baf60..81f0da60 100644 --- a/tests/test_get_all_scripts_recursively.py +++ b/tests/test_get_all_scripts_recursively.py @@ -13,7 +13,7 @@ def test_get_all_scripts_recursively__given_empty_folder_should_return_empty(): with mock.patch("os.walk") as mockwalk: mockwalk.return_value = [] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert result == dict() @@ -25,7 +25,7 @@ def test_get_all_scripts_recursively__given_just_non_change_files_should_return_ ("subfolder", ("subfolder2"), ("something.sql",)), (f"subfolder{os.sep}subfolder2", (""), ("testing.py",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert result == dict() @@ -43,7 +43,7 @@ def test_get_all_scripts_recursively__given_Version_files_should_return_version_ (f"subfolder{os.sep}subfolder2", (""), ("V1.1.3__update.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 3 assert "V1.1.1__intial.sql" in result @@ -60,7 +60,7 @@ def test_get_all_scripts_recursively__given_same_Version_twice_should_raise_exce ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( "The script version 1.1.1 exists more than once (second instance" ) @@ -71,7 +71,7 @@ def test_get_all_scripts_recursively__given_single_Version_file_should_extract_a mockwalk.return_value = [ ("subfolder", (), ("V1.1.1.1__THIS_is_my_test.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["V1.1.1.1__THIS_is_my_test.sql"] @@ -89,7 +89,7 @@ def test_get_all_scripts_recursively__given_single_Version_jinja_file_should_ext mockwalk.return_value = [ ("subfolder", (), ("V1.1.1.2__THIS_is_my_test.sql.jinja",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["V1.1.1.2__THIS_is_my_test.sql"] @@ -109,7 +109,7 @@ def test_get_all_scripts_recursively__given_same_version_file_with_and_without_j ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( "The script name V1.1.1__intial.sql exists more than once (first_instance" ) @@ -127,7 +127,7 @@ def test_get_all_scripts_recursively__given_Always_files_should_return_always_fi (f"subfolder{os.sep}subfolder2", (""), ("A__proc3.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 3 assert "A__proc1.sql" in result @@ -143,9 +143,9 @@ def test_get_all_scripts_recursively__given_same_Always_file_should_raise_except ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( - "The script name A__intial.sql exists more than once (first_instance " + "The script name A__intial.sql exists more than once (first_instance" ) @@ -154,7 +154,7 @@ def test_get_all_scripts_recursively__given_single_Always_file_should_extract_at mockwalk.return_value = [ ("subfolder", (), ("A__THIS_is_my_test.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["A__THIS_is_my_test.sql"] @@ -172,7 +172,7 @@ def test_get_all_scripts_recursively__given_single_Always_jinja_file_should_extr mockwalk.return_value = [ ("subfolder", (), ("A__THIS_is_my_test.sql.jinja",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["A__THIS_is_my_test.sql"] @@ -192,9 +192,9 @@ def test_get_all_scripts_recursively__given_same_Always_file_with_and_without_ji ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( - "The script name A__intial.sql exists more than once (first_instance " + "The script name A__intial.sql exists more than once (first_instance" ) ############################### @@ -210,7 +210,7 @@ def test_get_all_scripts_recursively__given_Repeatable_files_should_return_repea (f"subfolder{os.sep}subfolder2", (), ("R__proc3.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 3 assert "R__proc1.sql" in result @@ -226,9 +226,9 @@ def test_get_all_scripts_recursively__given_same_Repeatable_file_should_raise_ex ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( - "The script name R__intial.sql exists more than once (first_instance " + "The script name R__intial.sql exists more than once (first_instance" ) @@ -237,7 +237,7 @@ def test_get_all_scripts_recursively__given_single_Repeatable_file_should_extrac mockwalk.return_value = [ ("subfolder", (), ("R__THIS_is_my_test.sql",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["R__THIS_is_my_test.sql"] @@ -255,7 +255,7 @@ def test_get_all_scripts_recursively__given_single_Repeatable_jinja_file_should_ mockwalk.return_value = [ ("subfolder", (), ("R__THIS_is_my_test.sql.jinja",)), ] - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert len(result) == 1 file_attributes = result["R__THIS_is_my_test.sql"] @@ -275,7 +275,103 @@ def test_get_all_scripts_recursively__given_same_Repeatable_file_with_and_withou ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + result = get_all_scripts_recursively("scripts", False, False) assert str(e.value).startswith( - "The script name R__intial.sql exists more than once (first_instance " + "The script name R__intial.sql exists more than once (first_instance" + ) + +################################# +#### Always First file tests #### +################################# + + +def test_get_all_scripts_recursively__given_Always_First_files_should_return_Always_First_files(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("", ("subfolder"), ("F__QA_Clone.sql",)), + ("subfolder", ("subfolder2"), ("F__STG_Clone.SQL",)), + (f"subfolder{os.sep}subfolder2", (""), ("F__PermissionsGrants.sql",)), + ] + + result = get_all_scripts_recursively("scripts", False, True) + + assert len(result) == 3 + assert "F__QA_Clone.sql" in result + assert "F__STG_Clone.SQL" in result + assert "F__PermissionsGrants.sql" in result + + +def test_get_all_scripts_recursively__given_Always_First_files_should_return_empty(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("", ("subfolder"), ("F__QA_Clone.sql",)), + ("subfolder", ("subfolder2"), ("F__STG_Clone.SQL",)), + (f"subfolder{os.sep}subfolder2", (""), ("F__PermissionsGrants.sql",)), + ] + + result = get_all_scripts_recursively("scripts", False, False) + + assert result == dict() + + +def test_get_all_scripts_recursively__given_same_Always_First_file_should_raise_exception(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("", ("subfolder"), ("F__QA_Clone.sql",)), + ("subfolder", (), ("F__QA_Clone.sql",)), + ] + + with pytest.raises(ValueError) as e: + result = get_all_scripts_recursively("scripts", False, True) + assert str(e.value).startswith( + "The script name F__QA_Clone.sql exists more than once (first_instance" + ) + + +def test_get_all_scripts_recursively__given_single_Always_First_file_should_extract_attributes(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("subfolder", (), ("F__THIS_is_my_test.sql",)), + ] + result = get_all_scripts_recursively("scripts", False, True) + + assert len(result) == 1 + file_attributes = result["F__THIS_is_my_test.sql"] + assert file_attributes["script_name"] == "F__THIS_is_my_test.sql" + assert file_attributes["script_full_path"] == os.path.join( + "subfolder", "F__THIS_is_my_test.sql" + ) + assert file_attributes["script_type"] == "F" + assert file_attributes["script_version"] == "" + assert file_attributes["script_description"] == "This is my test" + + +def test_get_all_scripts_recursively__given_single_Always_First_jinja_file_should_extract_attributes(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("subfolder", (), ("F__THIS_is_my_test.sql.jinja",)), + ] + result = get_all_scripts_recursively("scripts", False, True) + + assert len(result) == 1 + file_attributes = result["F__THIS_is_my_test.sql"] + assert file_attributes["script_name"] == "F__THIS_is_my_test.sql" + assert file_attributes["script_full_path"] == os.path.join( + "subfolder", "F__THIS_is_my_test.sql.jinja" + ) + assert file_attributes["script_type"] == "F" + assert file_attributes["script_version"] == "" + assert file_attributes["script_description"] == "This is my test" + + +def test_get_all_scripts_recursively__given_same_Always_First_file_with_and_without_jinja_extension_should_raise_exception(): + with mock.patch("os.walk") as mockwalk: + mockwalk.return_value = [ + ("", (""), ("F__QA_Clone.sql", "F__QA_Clone.sql.jinja")), + ] + + with pytest.raises(ValueError) as e: + result = get_all_scripts_recursively("scripts", False, True) + assert str(e.value).startswith( + "The script name F__QA_Clone.sql exists more than once (first_instance" ) diff --git a/tests/test_jinja_env_var_template.py b/tests/test_jinja_env_var_template.py index 52f26b02..ae4d1f5f 100644 --- a/tests/test_jinja_env_var_template.py +++ b/tests/test_jinja_env_var_template.py @@ -14,7 +14,7 @@ def test_from_environ_not_set(): processor.override_loader(DictLoader(templates)) with pytest.raises(ValueError) as e: - context = processor.render("test.sql", None, True) + context = processor.render("test.sql", None, True, False) assert str(e.value) == "Could not find environmental variable MYVAR and no default value was provided" @@ -29,7 +29,7 @@ def test_from_environ_set(): templates = {"test.sql": "some text {{ env_var('MYVAR') }}"} processor.override_loader(DictLoader(templates)) - context = processor.render("test.sql", None, True) + context = processor.render("test.sql", None, True, False) # unset MYVAR env variable del os.environ["MYVAR"] @@ -44,6 +44,6 @@ def test_from_environ_not_set_default(): templates = {"test.sql": "some text {{ env_var('MYVAR', 'myvar_default') }}"} processor.override_loader(DictLoader(templates)) - context = processor.render("test.sql", None, True) + context = processor.render("test.sql", None, True, False) assert context == "some text myvar_default" diff --git a/tests/test_main.py b/tests/test_main.py index 39217fe3..1ea1a955 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -21,6 +21,7 @@ 'create_change_history_table': False, 'autocommit': False, 'verbose': False, + 'always_first': False, 'dry_run': False, 'query_tag': None, 'oauth_config':None, @@ -54,6 +55,8 @@ {**DEFAULT_CONFIG, 'autocommit': True}), (["schemachange", "deploy", "--verbose"], {**DEFAULT_CONFIG, 'verbose': True}), + (["schemachange", "deploy", "--always-first"], + {**DEFAULT_CONFIG, 'always_first': True}), (["schemachange", "deploy", "--dry-run"], {**DEFAULT_CONFIG, 'dry_run': True}), (["schemachange", "deploy", "--query-tag", "querytag"], @@ -79,6 +82,8 @@ def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")), (["schemachange", "render", "--verbose", "script.sql"], ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")), + (["schemachange", "render", "--always-first", "script.sql"], + ({**DEFAULT_CONFIG, 'always_first': True}, "script.sql")), ]) def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected):