Feat: publish all docs in folder

contributing.md

@@ -106,6 +106,15 @@ _(note that running mypy and black with no errors is required before code will b
 - do test coverage calculation (https://coverage.readthedocs.io/en/6.3.2)
 > bin/coverage.sh
 
+### Testing
+
+We have three levels of testing.
+1. Testing for all the little helper methods. These should be straightforward unit tests.
+
+2. "e2e" offline testing: Every command needs to have test coverage calling "MyCommand.run_command()" that will run through the basic happy path of the command. Since they are offline, they need to use the mocked server object set up in mock_data. These tests are in files named test_e2e_x_command.py, and they are run with the unit tests against every checkin.
+
+3. real, live, e2e tests that you can run against a known server when given credentials. These tests should not have any mocking code. You can launch these tests manually (see above)
+
 
 ### Packaging 
 - build an executable package with [pyinstaller](https://github.com/pyinstaller/pyinstaller). 

tabcmd/commands/datasources_and_workbooks/publish_command.py

@@ -1,5 +1,6 @@
 import tableauserverclient as TSC
-from tableauserverclient import ServerResponseError
+import glob
+import os
 
 from tabcmd.commands.auth.session import Session
 from tabcmd.commands.constants import Errors
@@ -24,9 +25,13 @@ def define_args(publish_parser):
         group = publish_parser.add_argument_group(title=PublishCommand.name)
         group.add_argument(
             "filename",
+            # this is a string and not actually a File type because we just pass the path to tsc
             metavar="filename.twbx|tdsx|hyper",
-            # this is not actually a File type because we just pass the path to tsc
+            help="The specified file to publish. If a folder is given, it will publish all files in this folder \
+                that have the extensions twb, twbx, tdsx or hyper. Any other options set will be applied for all files."
         )
+        group.add_argument("--filetype", metavar="twb|twbx|tdxs|hyper", help="If publishing an entire folder, limit files to this filetype.")
+        group.add_argument("--recursive", help="If publishing an entire folder, look into subdirectories to find files")
         set_publish_args(group)
         set_project_r_arg(group)
         set_overwrite_option(group)
@@ -61,42 +66,77 @@ def run_command(args):
         else:
             logger.debug("No db-username or oauth-username found in command")
             creds = None
+        credentials = TSC.ConnectionItem() if creds else None
+        if credentials:
+            credentials.connection_credentials = creds
+
+        files = PublishCommand.get_files_to_publish(args, logger)
+
+        logger.debug("Publishing {} files".format(len(files)))
+        for str_filename in files:
+            source = PublishCommand.get_filename_extension_if_tableau_type(logger, str_filename)
+            logger.info(_("publish.status").format(str_filename))
+            if source in ["twbx", "twb"]:
+                if args.thumbnail_group:
+                    raise AttributeError("Generating thumbnails for a group is not yet implemented.")
+                if args.thumbnail_username and args.thumbnail_group:
+                    raise AttributeError("Cannot specify both a user and group for thumbnails.")
+
+                new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed)
+                try:
+                    new_workbook = server.workbooks.publish(
+                        new_workbook,
+                        str_filename,
+                        publish_mode,
+                        # args.thumbnail_username, not yet implemented in tsc
+                        # args.thumbnail_group,
+                        connections=credentials,
+                        as_job=False,
+                        skip_connection_check=args.skip_connection_check,
+                    )
+                except Exception as e:
+                    Errors.exit_with_error(logger, exception=e)
+
+                logger.info(_("publish.success") + "\n{}".format(new_workbook.webpage_url))
+
+            elif source in ["tds", "tdsx", "hyper"]:
+                new_datasource = TSC.DatasourceItem(project_id, name=args.name)
+                new_datasource.use_remote_query_agent = args.use_tableau_bridge
+                try:
+                    new_datasource = server.datasources.publish(
+                        new_datasource, str_filename, publish_mode, connection_credentials=creds
+                    )
+                except Exception as exc:
+                    Errors.exit_with_error(logger, exception=exc)
+                logger.info(_("publish.success") + "\n{}".format(new_datasource.webpage_url))
 
-        source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename)
-        logger.info(_("publish.status").format(args.filename))
-        if source in ["twbx", "twb"]:
-            if args.thumbnail_group:
-                raise AttributeError("Generating thumbnails for a group is not yet implemented.")
-            if args.thumbnail_username and args.thumbnail_group:
-                raise AttributeError("Cannot specify both a user and group for thumbnails.")
-
-            new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed)
-            try:
-                new_workbook = server.workbooks.publish(
-                    new_workbook,
-                    args.filename,
-                    publish_mode,
-                    # args.thumbnail_username, not yet implemented in tsc
-                    # args.thumbnail_group,
-                    connection_credentials=creds,
-                    as_job=False,
-                    skip_connection_check=args.skip_connection_check,
-                )
-            except Exception as e:
-                Errors.exit_with_error(logger, exception=e)
-
-            logger.info(_("publish.success") + "\n{}".format(new_workbook.webpage_url))
-
-        elif source in ["tds", "tdsx", "hyper"]:
-            new_datasource = TSC.DatasourceItem(project_id, name=args.name)
-            new_datasource.use_remote_query_agent = args.use_tableau_bridge
-            try:
-                new_datasource = server.datasources.publish(
-                    new_datasource, args.filename, publish_mode, connection_credentials=creds
-                )
-            except Exception as exc:
-                Errors.exit_with_error(logger, exception=exc)
-            logger.info(_("publish.success") + "\n{}".format(new_datasource.webpage_url))
+    @staticmethod
+    def get_files_to_publish(args, logger):
+        logger.debug("Checking file argument: {}".format(args.filename))
+        files = set()
+        if not os.path.exists(args.filename):
+            logger.debug("Invalid file")
+            Errors.exit_with_error(logger, message="Filename given does not exist: {}".format(args.filename))
+        elif os.path.isfile(args.filename):
+            logger.debug("Valid single file found")
+            files.add(args.filename)
+        elif os.path.isdir(args.filename):            
+            logger.debug("Valid folder found")
+            if args.filetype:
+                file_patterns = [args.filetype]
+            else:
+                file_patterns = ['*.twb?', '*.tdsx', '*.hyper']
+            logger.debug("file patterns: {}".format(file_patterns))
+            for file_pattern in file_patterns:
+                logger.debug("Looking for files {} in {}".format(file_pattern, args.filename))
+                try:
+                    in_place_files = (glob.glob(file_pattern, root_dir=args.filename, recursive=args.recursive, include_hidden=False))
+                    relative_files = list(map(lambda file: os.path.join(args.filename, file), in_place_files))
+                except Exception as e:            
+                    Errors.exit_with_error(logger, message=in_place_files)
+                files.update(relative_files)
+                logger.debug(len(files))
+        return sorted(files)
 
     # todo write tests for this method
     @staticmethod

tests/assets/mock_data.py

@@ -0,0 +1,104 @@
+import argparse
+import io
+
+import tableauserverclient as TSC
+from typing import NamedTuple, TextIO, Union
+from unittest.mock import *
+
+
+def create_fake_item():
+    fake_item = MagicMock()
+    fake_item.name = "fake-name"
+    fake_item.id = "fake-id"
+    fake_item.pdf = b"/pdf-representation-of-view"
+    fake_item.extract_encryption_mode = "Disabled"
+    return fake_item
+
+def create_fake_job():
+    fake_job = MagicMock()
+    fake_job.id = "fake-job-id"
+    return fake_job
+
+def set_up_mock_args():
+    mock_args = argparse.Namespace()
+    # auth/connection values
+    mock_args.timeout = None
+    mock_args.username = None
+    mock_args.server = None
+    mock_args.password_file = None
+    mock_args.token_file = None
+    mock_args.token_name = None
+    mock_args.token_value = None
+    mock_args.no_prompt = False
+    mock_args.certificate = None
+    mock_args.no_certcheck = True
+    mock_args.no_proxy = True
+    mock_args.proxy = None
+    mock_args.password = None
+    mock_args.site_name = None
+
+    # these are just really common
+    mock_args.project_name = None
+    mock_args.parent_project_path = None
+    mock_args.parent_path = None
+    mock_args.continue_if_exists = False    
+    mock_args.recursive = False
+    mock_args.logging_level="DEBUG"
+    return mock_args
+
+
+# TODO: get typings for argparse
+class NamedObject(NamedTuple):
+    name: str
+ArgparseFile = Union[TextIO, NamedObject]
+
+def set_up_mock_file(content=["Test", "", "Test", ""]) -> ArgparseFile:
+    # the empty string represents EOF
+    # the tests run through the file twice, first to validate then to fetch
+    mock = MagicMock(io.TextIOWrapper)
+    mock.readline.side_effect = content
+    mock.name = "file-mock"
+    return mock
+
+def set_up_mock_path(mock_path):
+    mock_path.exists = lambda x: True
+    mock_path.isfile = lambda x: True
+    mock_path.isdir = lambda x: True 
+    mock_path.splitext = lambda x: ['file', 'twbx']
+    mock_path.join = lambda x, y: x + "/" + y
+    mock_path.basename = lambda x: str(x)
+    return mock_path
+
+
+def set_up_mock_server(mock_session):
+
+    mock_session.return_value = mock_session
+    mock_server = MagicMock(TSC.Server, autospec=True)
+    getter = MagicMock()
+    # basically we want to mock out everything in TSC 
+    getter.get = MagicMock("get anything", return_value=([create_fake_item()], 1))
+    getter.publish = MagicMock("publish", return_value=create_fake_item())
+
+    mock_server.any_item_type = getter
+    mock_server.flows = getter
+    mock_server.groups = getter
+    mock_server.projects = getter
+    mock_server.sites = getter
+    mock_server.users = getter
+    mock_server.views = getter
+    mock_server.workbooks = getter
+
+    fake_job = create_fake_job()
+    # ideally I would only set these on the specific objects that have each action, but this is a start
+    getter.create_extract = MagicMock("create_extract", return_value=fake_job)
+    getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job)
+    getter.delete_extract = MagicMock("delete_extract", return_value=fake_job)
+    getter.encrypt_extracts = MagicMock("encrypt_extracts", return_value=fake_job)
+    getter.reencrypt_extract = MagicMock("reencrypt_extract", return_value=fake_job)
+    getter.refresh = MagicMock("refresh", return_value=fake_job)
+
+    # for test access
+    mock_session.internal_server = mock_server
+    mock_session.create_session.return_value = mock_server
+
+    return mock_session

tests/commands/test_publish_command.py

@@ -1,58 +1,26 @@
-import argparse
+import logging
 import unittest
-from unittest.mock import *
 import tableauserverclient as TSC
-
-from tabcmd.commands.auth import login_command
-from tabcmd.commands.datasources_and_workbooks import delete_command, export_command, get_url_command, publish_command
-
-
-from typing import List, NamedTuple, TextIO, Union
-import io
-
-mock_args = argparse.Namespace()
-
-fake_item = MagicMock()
-fake_item.name = "fake-name"
-fake_item.id = "fake-id"
-fake_item.pdf = b"/pdf-representation-of-view"
-fake_item.extract_encryption_mode = "Disabled"
-
-fake_item_pagination = MagicMock()
-fake_item_pagination.page_number = 1
-fake_item_pagination.total_available = 1
-fake_item_pagination.page_size = 100
-
-fake_job = MagicMock()
-fake_job.id = "fake-job-id"
-
-creator = MagicMock()
-getter = MagicMock()
-getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination))
-getter.publish = MagicMock("publish", return_value=fake_item)
-
-
-@patch("tableauserverclient.Server")
-@patch("tabcmd.commands.auth.session.Session.create_session")
-class RunCommandsTest(unittest.TestCase):
-    @staticmethod
-    def _set_up_session(mock_session, mock_server):
-        mock_session.return_value = mock_server
-        assert mock_session is not None
-        mock_session.assert_not_called()
-        global mock_args
-        mock_args = argparse.Namespace(logging_level="DEBUG")
-        # set values for things that should always have a default
-        # should refactor so this can be automated
-        mock_args.continue_if_exists = False
-        mock_args.project_name = None
-        mock_args.parent_project_path = None
-        mock_args.parent_path = None
-        mock_args.timeout = None
-        mock_args.username = None
-
-    def test_publish(self, mock_session, mock_server):
-        RunCommandsTest._set_up_session(mock_session, mock_server)
+from ..assets import mock_data
+from unittest.mock import *
+from tabcmd.commands.datasources_and_workbooks.publish_command import PublishCommand
+
+from ..assets.mock_data import set_up_mock_args, set_up_mock_file, set_up_mock_path, set_up_mock_server
+
+mock_args = set_up_mock_args()
+
+
+# mock the module as it is imported *when used*
+@patch("tabcmd.commands.datasources_and_workbooks.publish_command.Session", autospec=True)
+@patch("tabcmd.commands.datasources_and_workbooks.publish_command.glob.glob", return_value=["one.twbx", "two.hyper"])
+@patch("tabcmd.commands.datasources_and_workbooks.publish_command.os.path", autospec=True)
+class PublishCommandTests(unittest.TestCase):
+
+    def test_publish(self, mock_path, mock_glob, mock_session):
+        # TODO move to init method
+        set_up_mock_server(mock_session)
+        mock_path = set_up_mock_path(mock_path)
+
         mock_args.overwrite = False
         mock_args.filename = "existing_file.twbx"
         mock_args.project_name = "project-name"
@@ -66,12 +34,14 @@ def test_publish(self, mock_session, mock_server):
         mock_args.thumbnail_username = None
         mock_args.thumbnail_group = None
         mock_args.skip_connection_check = False
-        mock_server.projects = getter
-        publish_command.PublishCommand.run_command(mock_args)
-        mock_session.assert_called()
+        PublishCommand.run_command(mock_args)
+        mock_session.internal_server.workbooks.publish.assert_called()
+
 
-    def test_publish_with_creds(self, mock_session, mock_server):
-        RunCommandsTest._set_up_session(mock_session, mock_server)
+    def test_publish_with_creds(self, mock_path, mock_glob, mock_session):
+        set_up_mock_server(mock_session)
+        mock_path = set_up_mock_path(mock_path)
+
         mock_args.overwrite = False
         mock_args.append = True
         mock_args.replace = False
@@ -92,6 +62,27 @@ def test_publish_with_creds(self, mock_session, mock_server):
         mock_args.thumbnail_group = None
         mock_args.skip_connection_check = False
 
-        mock_server.projects = getter
-        publish_command.PublishCommand.run_command(mock_args)
-        mock_session.assert_called()
+        PublishCommand.run_command(mock_args)
+        mock_session.internal_server.workbooks.publish.assert_called()
+
+
+    def test_get_files_to_publish_twbx(self, mock_path, mock_glob, mock_session):
+        set_up_mock_server(mock_session)
+        mock_path = set_up_mock_path(mock_path)
+        mock_args.filename = "existing.twbx"
+        expected = ["existing.twbx"]
+        actual = PublishCommand.get_files_to_publish(mock_args, logging)
+        assert actual == expected
+
+    # if the filename given is a directory, publish all relevant files in the directory to the server
+    def test_get_files_to_publish_folder(self, mock_path, mock_glob, mock_session):
+        set_up_mock_server(mock_session)
+        mock_path = set_up_mock_path(mock_path)
+        mock_path.isfile = lambda x: False # this time it is a directory
+
+        mock_args.filename = "directory"        
+        mock_args.filetype = None
+        expected = ["directory/one.twbx", "directory/two.hyper"]
+        actual = PublishCommand.get_files_to_publish(mock_args, logging)
+        assert actual == expected
+