diff --git a/contributing.md b/contributing.md index 91b3a40a..4f987758 100644 --- a/contributing.md +++ b/contributing.md @@ -106,6 +106,15 @@ _(note that running mypy and black with no errors is required before code will b - do test coverage calculation (https://coverage.readthedocs.io/en/6.3.2) > bin/coverage.sh +### Testing + +We have three levels of testing. +1. Testing for all the little helper methods. These should be straightforward unit tests. + +2. "e2e" offline testing: Every command needs to have test coverage calling "MyCommand.run_command()" that will run through the basic happy path of the command. Since they are offline, they need to use the mocked server object set up in mock_data. These tests are in files named test_e2e_x_command.py, and they are run with the unit tests against every checkin. + +3. real, live, e2e tests that you can run against a known server when given credentials. These tests should not have any mocking code. You can launch these tests manually (see above) + ### Packaging - build an executable package with [pyinstaller](https://github.com/pyinstaller/pyinstaller). diff --git a/tabcmd/commands/datasources_and_workbooks/publish_command.py b/tabcmd/commands/datasources_and_workbooks/publish_command.py index a4ff8c26..571fd5f4 100644 --- a/tabcmd/commands/datasources_and_workbooks/publish_command.py +++ b/tabcmd/commands/datasources_and_workbooks/publish_command.py @@ -1,5 +1,6 @@ import tableauserverclient as TSC -from tableauserverclient import ServerResponseError +import glob +import os from tabcmd.commands.auth.session import Session from tabcmd.commands.constants import Errors @@ -24,9 +25,13 @@ def define_args(publish_parser): group = publish_parser.add_argument_group(title=PublishCommand.name) group.add_argument( "filename", + # this is a string and not actually a File type because we just pass the path to tsc metavar="filename.twbx|tdsx|hyper", - # this is not actually a File type because we just pass the path to tsc + help="The specified file to publish. If a folder is given, it will publish all files in this folder \ + that have the extensions twb, twbx, tdsx or hyper. Any other options set will be applied for all files." ) + group.add_argument("--filetype", metavar="twb|twbx|tdxs|hyper", help="If publishing an entire folder, limit files to this filetype.") + group.add_argument("--recursive", help="If publishing an entire folder, look into subdirectories to find files") set_publish_args(group) set_project_r_arg(group) set_overwrite_option(group) @@ -61,42 +66,77 @@ def run_command(args): else: logger.debug("No db-username or oauth-username found in command") creds = None + credentials = TSC.ConnectionItem() if creds else None + if credentials: + credentials.connection_credentials = creds + + files = PublishCommand.get_files_to_publish(args, logger) + + logger.debug("Publishing {} files".format(len(files))) + for str_filename in files: + source = PublishCommand.get_filename_extension_if_tableau_type(logger, str_filename) + logger.info(_("publish.status").format(str_filename)) + if source in ["twbx", "twb"]: + if args.thumbnail_group: + raise AttributeError("Generating thumbnails for a group is not yet implemented.") + if args.thumbnail_username and args.thumbnail_group: + raise AttributeError("Cannot specify both a user and group for thumbnails.") + + new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed) + try: + new_workbook = server.workbooks.publish( + new_workbook, + str_filename, + publish_mode, + # args.thumbnail_username, not yet implemented in tsc + # args.thumbnail_group, + connections=credentials, + as_job=False, + skip_connection_check=args.skip_connection_check, + ) + except Exception as e: + Errors.exit_with_error(logger, exception=e) + + logger.info(_("publish.success") + "\n{}".format(new_workbook.webpage_url)) + + elif source in ["tds", "tdsx", "hyper"]: + new_datasource = TSC.DatasourceItem(project_id, name=args.name) + new_datasource.use_remote_query_agent = args.use_tableau_bridge + try: + new_datasource = server.datasources.publish( + new_datasource, str_filename, publish_mode, connection_credentials=creds + ) + except Exception as exc: + Errors.exit_with_error(logger, exception=exc) + logger.info(_("publish.success") + "\n{}".format(new_datasource.webpage_url)) - source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename) - logger.info(_("publish.status").format(args.filename)) - if source in ["twbx", "twb"]: - if args.thumbnail_group: - raise AttributeError("Generating thumbnails for a group is not yet implemented.") - if args.thumbnail_username and args.thumbnail_group: - raise AttributeError("Cannot specify both a user and group for thumbnails.") - - new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed) - try: - new_workbook = server.workbooks.publish( - new_workbook, - args.filename, - publish_mode, - # args.thumbnail_username, not yet implemented in tsc - # args.thumbnail_group, - connection_credentials=creds, - as_job=False, - skip_connection_check=args.skip_connection_check, - ) - except Exception as e: - Errors.exit_with_error(logger, exception=e) - - logger.info(_("publish.success") + "\n{}".format(new_workbook.webpage_url)) - - elif source in ["tds", "tdsx", "hyper"]: - new_datasource = TSC.DatasourceItem(project_id, name=args.name) - new_datasource.use_remote_query_agent = args.use_tableau_bridge - try: - new_datasource = server.datasources.publish( - new_datasource, args.filename, publish_mode, connection_credentials=creds - ) - except Exception as exc: - Errors.exit_with_error(logger, exception=exc) - logger.info(_("publish.success") + "\n{}".format(new_datasource.webpage_url)) + @staticmethod + def get_files_to_publish(args, logger): + logger.debug("Checking file argument: {}".format(args.filename)) + files = set() + if not os.path.exists(args.filename): + logger.debug("Invalid file") + Errors.exit_with_error(logger, message="Filename given does not exist: {}".format(args.filename)) + elif os.path.isfile(args.filename): + logger.debug("Valid single file found") + files.add(args.filename) + elif os.path.isdir(args.filename): + logger.debug("Valid folder found") + if args.filetype: + file_patterns = [args.filetype] + else: + file_patterns = ['*.twb?', '*.tdsx', '*.hyper'] + logger.debug("file patterns: {}".format(file_patterns)) + for file_pattern in file_patterns: + logger.debug("Looking for files {} in {}".format(file_pattern, args.filename)) + try: + in_place_files = (glob.glob(file_pattern, root_dir=args.filename, recursive=args.recursive, include_hidden=False)) + relative_files = list(map(lambda file: os.path.join(args.filename, file), in_place_files)) + except Exception as e: + Errors.exit_with_error(logger, message=in_place_files) + files.update(relative_files) + logger.debug(len(files)) + return sorted(files) # todo write tests for this method @staticmethod diff --git a/tests/assets/mock_data.py b/tests/assets/mock_data.py new file mode 100644 index 00000000..633c5528 --- /dev/null +++ b/tests/assets/mock_data.py @@ -0,0 +1,104 @@ +import argparse +import io + +import tableauserverclient as TSC +from typing import NamedTuple, TextIO, Union +from unittest.mock import * + + +def create_fake_item(): + fake_item = MagicMock() + fake_item.name = "fake-name" + fake_item.id = "fake-id" + fake_item.pdf = b"/pdf-representation-of-view" + fake_item.extract_encryption_mode = "Disabled" + return fake_item + +def create_fake_job(): + fake_job = MagicMock() + fake_job.id = "fake-job-id" + return fake_job + +def set_up_mock_args(): + mock_args = argparse.Namespace() + # auth/connection values + mock_args.timeout = None + mock_args.username = None + mock_args.server = None + mock_args.password_file = None + mock_args.token_file = None + mock_args.token_name = None + mock_args.token_value = None + mock_args.no_prompt = False + mock_args.certificate = None + mock_args.no_certcheck = True + mock_args.no_proxy = True + mock_args.proxy = None + mock_args.password = None + mock_args.site_name = None + + # these are just really common + mock_args.project_name = None + mock_args.parent_project_path = None + mock_args.parent_path = None + mock_args.continue_if_exists = False + mock_args.recursive = False + mock_args.logging_level="DEBUG" + return mock_args + + +# TODO: get typings for argparse +class NamedObject(NamedTuple): + name: str +ArgparseFile = Union[TextIO, NamedObject] + +def set_up_mock_file(content=["Test", "", "Test", ""]) -> ArgparseFile: + # the empty string represents EOF + # the tests run through the file twice, first to validate then to fetch + mock = MagicMock(io.TextIOWrapper) + mock.readline.side_effect = content + mock.name = "file-mock" + return mock + +def set_up_mock_path(mock_path): + mock_path.exists = lambda x: True + mock_path.isfile = lambda x: True + mock_path.isdir = lambda x: True + mock_path.splitext = lambda x: ['file', 'twbx'] + mock_path.join = lambda x, y: x + "/" + y + mock_path.basename = lambda x: str(x) + return mock_path + + +def set_up_mock_server(mock_session): + + mock_session.return_value = mock_session + mock_server = MagicMock(TSC.Server, autospec=True) + getter = MagicMock() + # basically we want to mock out everything in TSC + getter.get = MagicMock("get anything", return_value=([create_fake_item()], 1)) + getter.publish = MagicMock("publish", return_value=create_fake_item()) + + mock_server.any_item_type = getter + mock_server.flows = getter + mock_server.groups = getter + mock_server.projects = getter + mock_server.sites = getter + mock_server.users = getter + mock_server.views = getter + mock_server.workbooks = getter + + fake_job = create_fake_job() + # ideally I would only set these on the specific objects that have each action, but this is a start + getter.create_extract = MagicMock("create_extract", return_value=fake_job) + getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job) + getter.delete_extract = MagicMock("delete_extract", return_value=fake_job) + getter.encrypt_extracts = MagicMock("encrypt_extracts", return_value=fake_job) + getter.reencrypt_extract = MagicMock("reencrypt_extract", return_value=fake_job) + getter.refresh = MagicMock("refresh", return_value=fake_job) + + # for test access + mock_session.internal_server = mock_server + mock_session.create_session.return_value = mock_server + + return mock_session diff --git a/tests/commands/test_publish_command.py b/tests/commands/test_publish_command.py index 800c81fc..da388f51 100644 --- a/tests/commands/test_publish_command.py +++ b/tests/commands/test_publish_command.py @@ -1,58 +1,26 @@ -import argparse +import logging import unittest -from unittest.mock import * import tableauserverclient as TSC - -from tabcmd.commands.auth import login_command -from tabcmd.commands.datasources_and_workbooks import delete_command, export_command, get_url_command, publish_command - - -from typing import List, NamedTuple, TextIO, Union -import io - -mock_args = argparse.Namespace() - -fake_item = MagicMock() -fake_item.name = "fake-name" -fake_item.id = "fake-id" -fake_item.pdf = b"/pdf-representation-of-view" -fake_item.extract_encryption_mode = "Disabled" - -fake_item_pagination = MagicMock() -fake_item_pagination.page_number = 1 -fake_item_pagination.total_available = 1 -fake_item_pagination.page_size = 100 - -fake_job = MagicMock() -fake_job.id = "fake-job-id" - -creator = MagicMock() -getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) -getter.publish = MagicMock("publish", return_value=fake_item) - - -@patch("tableauserverclient.Server") -@patch("tabcmd.commands.auth.session.Session.create_session") -class RunCommandsTest(unittest.TestCase): - @staticmethod - def _set_up_session(mock_session, mock_server): - mock_session.return_value = mock_server - assert mock_session is not None - mock_session.assert_not_called() - global mock_args - mock_args = argparse.Namespace(logging_level="DEBUG") - # set values for things that should always have a default - # should refactor so this can be automated - mock_args.continue_if_exists = False - mock_args.project_name = None - mock_args.parent_project_path = None - mock_args.parent_path = None - mock_args.timeout = None - mock_args.username = None - - def test_publish(self, mock_session, mock_server): - RunCommandsTest._set_up_session(mock_session, mock_server) +from ..assets import mock_data +from unittest.mock import * +from tabcmd.commands.datasources_and_workbooks.publish_command import PublishCommand + +from ..assets.mock_data import set_up_mock_args, set_up_mock_file, set_up_mock_path, set_up_mock_server + +mock_args = set_up_mock_args() + + +# mock the module as it is imported *when used* +@patch("tabcmd.commands.datasources_and_workbooks.publish_command.Session", autospec=True) +@patch("tabcmd.commands.datasources_and_workbooks.publish_command.glob.glob", return_value=["one.twbx", "two.hyper"]) +@patch("tabcmd.commands.datasources_and_workbooks.publish_command.os.path", autospec=True) +class PublishCommandTests(unittest.TestCase): + + def test_publish(self, mock_path, mock_glob, mock_session): + # TODO move to init method + set_up_mock_server(mock_session) + mock_path = set_up_mock_path(mock_path) + mock_args.overwrite = False mock_args.filename = "existing_file.twbx" mock_args.project_name = "project-name" @@ -66,12 +34,14 @@ def test_publish(self, mock_session, mock_server): mock_args.thumbnail_username = None mock_args.thumbnail_group = None mock_args.skip_connection_check = False - mock_server.projects = getter - publish_command.PublishCommand.run_command(mock_args) - mock_session.assert_called() + PublishCommand.run_command(mock_args) + mock_session.internal_server.workbooks.publish.assert_called() + - def test_publish_with_creds(self, mock_session, mock_server): - RunCommandsTest._set_up_session(mock_session, mock_server) + def test_publish_with_creds(self, mock_path, mock_glob, mock_session): + set_up_mock_server(mock_session) + mock_path = set_up_mock_path(mock_path) + mock_args.overwrite = False mock_args.append = True mock_args.replace = False @@ -92,6 +62,27 @@ def test_publish_with_creds(self, mock_session, mock_server): mock_args.thumbnail_group = None mock_args.skip_connection_check = False - mock_server.projects = getter - publish_command.PublishCommand.run_command(mock_args) - mock_session.assert_called() + PublishCommand.run_command(mock_args) + mock_session.internal_server.workbooks.publish.assert_called() + + + def test_get_files_to_publish_twbx(self, mock_path, mock_glob, mock_session): + set_up_mock_server(mock_session) + mock_path = set_up_mock_path(mock_path) + mock_args.filename = "existing.twbx" + expected = ["existing.twbx"] + actual = PublishCommand.get_files_to_publish(mock_args, logging) + assert actual == expected + + # if the filename given is a directory, publish all relevant files in the directory to the server + def test_get_files_to_publish_folder(self, mock_path, mock_glob, mock_session): + set_up_mock_server(mock_session) + mock_path = set_up_mock_path(mock_path) + mock_path.isfile = lambda x: False # this time it is a directory + + mock_args.filename = "directory" + mock_args.filetype = None + expected = ["directory/one.twbx", "directory/two.hyper"] + actual = PublishCommand.get_files_to_publish(mock_args, logging) + assert actual == expected +