feat(cli): add export option to readiness assessment and update documentation links (#488)

* feat(cli): add export option to readiness assessment and update documentation links

* feat: test exports command

* feat: allow export directory  to be configured

* fix: build update

Author: cofin

.github/workflows/release.yaml

@@ -110,10 +110,6 @@ jobs:
         - target: x86_64-pc-windows-msvc
           os: windows-latest
           release_suffix: x86_64-windows
-        # - target: i686-pc-windows-msvc
-        #   os: windows-latest
-        #   release_suffix: i686-windows
-        # macOS
         - target: aarch64-apple-darwin
           os: macos-latest
           release_suffix: aarch64-osx
@@ -128,10 +124,10 @@ jobs:
       PYAPP_VERSION: v0.27.0
       PYAPP_PYTHON_VERSION: "3.12"
       PYAPP_PROJECT_FEATURES: oracle,postgres,mssql,mysql,server
-      PYAPP_DISTRIBUTION_EMBED: "1"
-      PYAPP_UV_ENABLED: "1"
+      PYAPP_DISTRIBUTION_EMBED: "true"
+      PYAPP_UV_ENABLED: "true"
       HATCH_BUILD_LOCATION: dist
-      PYAPP_FULL_ISOLATION: "1"
+      PYAPP_FULL_ISOLATION: "true"
     steps:
     - name: Checkout code
       uses: actions/checkout@v4

.gitignore

@@ -180,7 +180,8 @@ duck.db
 local.duckdb
 node_modules
 opdb__*
-
+assessment.db
+assessment.db.wal*
 src/dma/static
 src/dma/templates/*
 .astro

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
   # Ruff replaces black, flake8, autoflake and isort
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.9.10" # make sure this is always consistent with hatch configs
+    rev: "v0.11.1" # make sure this is always consistent with hatch configs
     hooks:
       - id: ruff
         args: [--config, ./pyproject.toml]

src/dma/cli/_utils.py

@@ -18,4 +18,3 @@
 __all__ = ("console",)
 
 console = get_console()
-console._width = 80

src/dma/cli/main.py

@@ -44,11 +44,6 @@ def app(ctx: Context) -> None:
     """Database Migration Assessment"""
 
 
-@app.command(
-    name="collect",
-    no_args_is_help=True,
-    short_help="Collect data from a source database.",
-)
 @click.option(
     "--no-prompt",
     help="Do not prompt for confirmation before executing check.",
@@ -168,10 +163,11 @@ def _collect_data(
     database: str,
     collection_identifier: str | None,
     working_path: Path | None = None,
+    export_path: Path | None = None,
+    export_delimiter: str = "|",
 ) -> None:
-    working_path = working_path or Path("tmp/")
     _execution_id = f"{src_info.db_type}_{current_version!s}_{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}"
-    with get_duckdb_connection(working_path) as local_db:
+    with get_duckdb_connection(working_path=working_path, export_path=export_path) as local_db:
         canonical_query_manager = next(provide_canonical_queries(local_db=local_db, working_path=working_path))
         collection_extractor = CollectionExtractor(
             local_db=local_db,
@@ -182,7 +178,9 @@ def _collect_data(
             collection_identifier=collection_identifier,
         )
         collection_extractor.execute()
-        collection_extractor.dump_database(working_path)
+        if collection_extractor is not None and export_path is not None:
+            collection_extractor.dump_database(export_path=export_path, delimiter=export_delimiter)
+        console.rule("Assessment complete.", align="left")
 
 
 @app.command(
@@ -262,6 +260,24 @@ def _collect_data(
     required=False,
     show_default=False,
 )
+@click.option(
+    "--export",
+    "-e",
+    help="Path to export the results.",
+    default=None,
+    type=click.Path(),
+    required=False,
+    show_default=False,
+)
+@click.option(
+    "--working-path",
+    "-wp",
+    help="Path to store the temporary artifacts during assessment.",
+    default=None,
+    type=click.Path(),
+    required=False,
+    show_default=False,
+)
 def readiness_assessment(
     no_prompt: bool,
     db_type: Literal["mysql", "postgres", "mssql", "oracle"],
@@ -271,6 +287,8 @@ def readiness_assessment(
     port: int | None = None,
     database: str | None = None,
     collection_identifier: str | None = None,
+    export: str | None = None,
+    working_path: str | None = None,
 ) -> None:
     """Process a collection of advisor extracts."""
     print_app_info()
@@ -298,6 +316,8 @@ def readiness_assessment(
             ),
             database=database,
             collection_identifier=collection_identifier,
+            working_path=Path(working_path) if working_path else None,
+            export_path=Path(export) if export else None,
         )
     else:
         console.rule("Skipping execution until input is confirmed", align="left")
@@ -309,10 +329,11 @@ def _readiness_check(
     database: str,
     collection_identifier: str | None,
     working_path: Path | None = None,
+    export_path: Path | None = None,
+    export_delimiter: str = "|",
 ) -> None:
-    working_path = working_path or Path("tmp/")
     _execution_id = f"{src_info.db_type}_{current_version!s}_{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}"
-    with get_duckdb_connection(working_path) as local_db:
+    with get_duckdb_connection(working_path=working_path, export_path=export_path) as local_db:
         workflow = ReadinessCheck(
             local_db=local_db,
             src_info=src_info,
@@ -325,8 +346,9 @@ def _readiness_check(
         console.print(Padding("", 1, expand=True))
         console.rule("Processing collected data.", align="left")
         workflow.print_summary()
-        if workflow.collection_extractor is not None:
-            workflow.collection_extractor.dump_database(working_path)
+        if workflow.collection_extractor is not None and export_path is not None:
+            workflow.collection_extractor.dump_database(export_path=export_path, delimiter=export_delimiter)
+        console.rule("Assessment complete.", align="left")
 
 
 def print_app_info() -> None:
@@ -335,4 +357,4 @@ def print_app_info() -> None:
     table.add_row(
         f"[bold green]Google Database Migration Assessment[/]                [cyan]version {current_version}[/]"
     )
-    console.print(table, width=80)
+    console.print(table)

src/dma/collector/dependencies.py

@@ -92,11 +92,13 @@ def provide_collection_query_manager(
 
 
 def provide_canonical_queries(
-    local_db: duckdb.DuckDBPyConnection | None = None, working_path: Path | None = None
+    local_db: duckdb.DuckDBPyConnection | None = None,
+    working_path: Path | None = None,
+    export_path: Path | None = None,
 ) -> Generator[CanonicalQueryManager, None, None]:
     """Construct repository and service objects for the request."""
     if local_db:
         yield CanonicalQueryManager(connection=local_db)
     else:
-        with get_duckdb_connection(working_path=working_path) as db_connection:
+        with get_duckdb_connection(working_path=working_path, export_path=export_path) as db_connection:
             yield CanonicalQueryManager(connection=db_connection)

src/dma/collector/workflows/readiness_check/_postgres/main.py

@@ -762,11 +762,12 @@ def table_for_target(migration_target: PostgresVariants) -> None:
             self.console.print(count_table)
             if migration_target == ALLOYDB:
                 self.console.print(
-                    "Please refer to https://cloud.google.com/database-migration/docs/postgresql-to-alloydb/configure-source-database for more details."
+                    "Please refer to the Alloy DB documentation for more details: https://cloud.google.com/database-migration/docs/postgresql-to-alloydb/configure-source-database"
                 )
             if migration_target == CLOUDSQL:
                 self.console.print(
-                    "Please refer to https://cloud.google.com/database-migration/docs/postgres/configure-source-database for more details."
+                    "Please refer to the CloudSQL documentation for more details: https://cloud.google.com/database-migration/docs/postgres/configure-source-database",
+                    markup=True,
                 )
 
         for v in db_variants:

src/dma/lib/db/local.py

@@ -25,8 +25,15 @@
 
 
 @contextmanager
-def get_duckdb_connection(working_path: Path | None = None) -> Iterator[duckdb.DuckDBPyConnection]:
+def get_duckdb_connection(
+    working_path: Path | None = None, export_path: Path | None = None, database: str | None = None
+) -> Iterator[duckdb.DuckDBPyConnection]:
     """Yield a new duckdb connections and automatically manages resource cleanup."""
+
+    if database is None and export_path is not None:
+        database = f"{Path(export_path / 'assessment.db').absolute()!s}"
+    elif database is None:
+        database = ":memory:"
     if working_path is None:
         working_path = Path(tempfile.gettempdir())
     config = {
@@ -37,15 +44,11 @@ def get_duckdb_connection(working_path: Path | None = None) -> Iterator[duckdb.D
     }
     Path(working_path).mkdir(parents=True, exist_ok=True)
     with duckdb.connect(
-        database=f"{working_path!s}/assessment.db",
+        database=database,
         read_only=False,
         config=config,
     ) as local_db:
         try:
-            # for extension in extensions:
-            """
-            local_db.execute("SET disabled_optimizers TO 'join_order'")
-            """
             yield local_db
         finally:
             local_db.close()