Minor updates for v0.10.7 (#148)

* + update version

* + precommit autoupdate

* + ensure import DaskParquetIO into curate namespace

* +improve describe

Author: jonsedar

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
       - id: no-print-statements
         files: ^oreum_core/
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.5
+    rev: v0.11.7
     hooks:
       - id: ruff   # lint
         args: [ --fix]

oreum_core/__init__.py

@@ -16,7 +16,7 @@
 
 import logging
 
-__version__ = "0.10.6"
+__version__ = "0.10.7"
 
 # logger goes to null handler by default
 # packages that import oreum_core can override this and direct elsewhere

oreum_core/curate/__init__.py

@@ -16,6 +16,7 @@
 """Various classes & functions for data curation"""
 
 from .data_io import (
+    DaskParquetIO,
     PandasCSVIO,
     PandasExcelIO,
     PandasParquetIO,

oreum_core/eda/describe.py

@@ -16,6 +16,7 @@
 """Data Descriptions"""
 
 import logging
+from copy import copy
 
 import numpy as np
 import pandas as pd
@@ -40,7 +41,7 @@ def describe(
     get_cr94: bool = False,
     reset_index: bool = True,
     return_df: bool = False,
-    subsample: bool = False,
+    subsample: bool = True,
     **kwargs,
 ) -> pd.DataFrame | None:
     """Concat transposed topN rows, numerical desc & dtypes
@@ -52,12 +53,13 @@ def describe(
     df = df.copy()
     len_idx = df.index.nlevels
     nbytes = df.values.nbytes
-    _log.info(f"Shape: {df.shape}")
+    shape = df.shape
+    _log.info(f"Shape: {shape}")
     _log.info(f"Memsize: {nbytes // 1e6:,.1f} MB")
     _log.info(f"Index levels: {df.index.names}")
-    if nfeats + len_idx < df.shape[1]:
+    if nfeats + len_idx < shape[1]:
         _log.info(
-            f"NOTE: nfeats + index shown {nfeats + len_idx}" + f" < width {df.shape[1]}"
+            f"NOTE: nfeats + index shown {nfeats + len_idx}" + f" < width {shape[1]}"
         )
 
     limit *= 1e6
@@ -67,6 +69,10 @@ def describe(
         )
         if subsample:
             df = df.sample(frac=(limit * 0.99) / nbytes, random_state=42)
+            nbytes_pre = copy(nbytes)
+            shape_pre = copy(shape)
+            nbytes = df.values.nbytes
+            shape = df.shape
             nobs = min(nobs, len(df))
             _log.info(txt + f", taking a subsample of {len(df)} rows")
         else:
@@ -157,20 +163,21 @@ def describe(
     if return_df:
         return dfout
     else:
-        display_fw(
-            dfout.iloc[: nfeats + len_idx, :],
-            max_rows=nfeats,
-            shape=df.shape,
-            nbytes=nbytes,
-            **kwargs,
-        )
+        kws_out = dict(max_rows=nfeats, shape=shape, nbytes=nbytes)
+        if subsample:
+            kws_out["txtadd"] = (
+                f"subsampled from Shape: {shape_pre},"
+                + f" Memsize {nbytes_pre / 1e6:,.1f} MB"
+            )
+        display_fw(dfout.iloc[: nfeats + len_idx, :], **kws_out, **kwargs)
 
 
 def display_fw(df: pd.DataFrame, **kwargs) -> None:
     """Conv fn: contextually display max cols"""
 
     shape = kwargs.pop("shape", df.shape)
     nbytes = kwargs.pop("nbytes", df.values.nbytes)
+    txtadd = kwargs.pop("txtadd", None)
 
     options = {
         "display.precision": kwargs.pop("precision", 2),
@@ -185,7 +192,8 @@ def display_fw(df: pd.DataFrame, **kwargs) -> None:
 
     with pd.option_context(*[i for tup in options.items() for i in tup]):
         display(df)
-        display(f"Shape: {shape}, Memsize {nbytes / 1e6:,.1f} MB")
+        t = f"Shape: {shape}, Memsize {nbytes / 1e6:,.1f} MB"
+        display(", ".join(filter(None, [t, txtadd])))
 
 
 def display_ht(df: pd.DataFrame, nrows=3, **kwargs) -> None: