feat(suite benchmarking): add progress tracking

examples/Advanced/suite_benchmarking_tutorial.py

@@ -0,0 +1,170 @@
+"""
+========================================
+Benchmarking with Progress Tracking
+========================================
+
+This tutorial demonstrates how to benchmark machine learning models
+across entire OpenML suites with real-time progress tracking.
+"""
+
+# License: BSD 3-Clause
+
+# %% [markdown]
+# ## Introduction
+#
+# The ``run_suite_with_progress`` function provides an easy way to benchmark
+# models across entire OpenML benchmark suites with real-time progress tracking
+# via the tqdm library.
+
+# %%
+import openml
+import openml_sklearn  # Required for sklearn models
+from sklearn.ensemble import RandomForestClassifier
+
+from openml.study import run_suite_with_progress
+
+# Configure to use test server for examples
+openml.config.start_using_configuration_for_example()
+
+# %% [markdown]
+# .. note::
+#     This example requires:
+#
+#     * ``pip install tqdm`` for progress bars
+#     * ``pip install openml-sklearn`` for sklearn model support
+#     * OpenML API key - get from https://test.openml.org (test server)
+#       or https://www.openml.org (main server)
+#
+#     Set your API key with:
+#
+#     >>> openml.config.apikey = 'YOURKEY'
+
+# %% [markdown]
+# ## Running a Benchmark with Progress Bar
+#
+# The progress bar shows real-time updates as tasks complete. When running,
+# you'll see something like:
+#
+# .. code-block:: text
+#
+#     Benchmarking Test Suite: 100%|████████| 3/3 [01:23<00:00, 27.8s/task]
+
+# %%
+# Create a simple model
+clf = RandomForestClassifier(n_estimators=10, max_depth=5, random_state=42)
+
+# Run benchmark on a test suite with progress bar
+# Note: This will fail without an API key
+try:
+    runs = run_suite_with_progress(
+        suite_id=99,  # Test suite ID on test server
+        model=clf,
+        show_progress=True,  # Shows progress bar (requires tqdm)
+    )
+    print(f"\nCompleted {len(runs)} tasks successfully!")
+except openml.exceptions.OpenMLServerException as e:
+    print(f"Server error (likely missing API key): {e}")
+    print("Set your API key with: openml.config.apikey = 'YOURKEY'")
+
+# %% [markdown]
+# ## Running Without Progress Bar
+#
+# You can disable the progress bar if tqdm is not installed, or if you
+# prefer cleaner output in log files.
+
+# %%
+try:
+    runs = run_suite_with_progress(
+        suite_id=99,
+        model=clf,
+        show_progress=False,  # No progress bar
+    )
+    print(f"Completed {len(runs)} tasks")
+except openml.exceptions.OpenMLServerException as e:
+    print(f"Server error: {e}")
+
+# %% [markdown]
+# ## Passing Additional Parameters
+#
+# All parameters from ``run_model_on_task`` are supported via ``**run_kwargs``:
+
+# %%
+try:
+    runs = run_suite_with_progress(
+        suite_id=99,
+        model=clf,
+        show_progress=True,
+        # Additional run_model_on_task parameters:
+        avoid_duplicate_runs=False,  # Allow re-running tasks
+        upload_flow=True,  # Upload the model flow
+    )
+except openml.exceptions.OpenMLServerException as e:
+    print(f"Server error: {e}")
+
+# %% [markdown]
+# ## Working with Results
+#
+# The function returns a list of ``OpenMLRun`` objects that can be analyzed:
+
+# %%
+# Example of working with results (when runs are successful)
+# for run in runs:
+#     print(f"Task {run.task_id}: Run ID {run.run_id}")
+#     # Access evaluations
+#     if run.evaluations:
+#         for metric, value in run.evaluations.items():
+#             print(f"  {metric}: {value}")
+
+# %% [markdown]
+# ## Common Use Cases
+#
+# **Quick Testing on Small Suites:**
+#
+# .. code-block:: python
+#
+#     import openml
+#     import openml_sklearn
+#     from sklearn.ensemble import RandomForestClassifier
+#     from openml.study import run_suite_with_progress
+#
+#     clf = RandomForestClassifier(random_state=42)
+#     # Suite 334: OpenML-Tiny (7 tasks) - on main server
+#     runs = run_suite_with_progress(suite_id=334, model=clf)
+#
+# **Production Benchmarking:**
+#
+# .. code-block:: python
+#
+#     # Suite 99: OpenML-CC18 (72 classification tasks) - on main server
+#     runs = run_suite_with_progress(suite_id=99, model=clf)
+#
+# **Comparing Multiple Models:**
+#
+# .. code-block:: python
+#
+#     from sklearn.linear_model import LogisticRegression
+#     from sklearn.tree import DecisionTreeClassifier
+#
+#     models = {
+#         'RandomForest': RandomForestClassifier(random_state=42),
+#         'LogisticRegression': LogisticRegression(random_state=42),
+#         'DecisionTree': DecisionTreeClassifier(random_state=42),
+#     }
+#
+#     results = {}
+#     for name, model in models.items():
+#         print(f"Benchmarking {name}...")
+#         results[name] = run_suite_with_progress(suite_id=334, model=model)
+
+# %% [markdown]
+# ## Requirements
+#
+# * ``tqdm`` package for progress bars: ``pip install tqdm``
+# * ``openml-sklearn`` extension: ``pip install openml-sklearn``
+# * OpenML API key (get from https://www.openml.org/auth/api-key)
+# * Internet connection to access OpenML server
+#
+# To disable progress bar if tqdm is not available, set ``show_progress=False``
+
+# %%
+openml.config.stop_using_configuration_for_example()

openml/study/__init__.py

@@ -1,5 +1,6 @@
 # License: BSD 3-Clause
 
+from .benchmarking import run_suite_with_progress
 from .functions import (
     attach_to_study,
     attach_to_suite,
@@ -19,8 +20,8 @@
 from .study import OpenMLBenchmarkSuite, OpenMLStudy
 
 __all__ = [
-    "OpenMLStudy",
     "OpenMLBenchmarkSuite",
+    "OpenMLStudy",
     "attach_to_study",
     "attach_to_suite",
     "create_benchmark_suite",
@@ -33,6 +34,7 @@
     "get_suite",
     "list_studies",
     "list_suites",
-    "update_suite_status",
+    "run_suite_with_progress",
     "update_study_status",
+    "update_suite_status",
 ]

openml/study/benchmarking.py

@@ -0,0 +1,89 @@
+# License: BSD 3-Clause
+
+"""Utilities for running benchmarks on OpenML suites with progress tracking."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import openml
+
+if TYPE_CHECKING:
+    from openml.runs import OpenMLRun
+
+try:
+    from tqdm.auto import tqdm
+
+    TQDM_AVAILABLE = True
+except ImportError:
+    TQDM_AVAILABLE = False
+
+
+def run_suite_with_progress(
+    suite_id: int | str,
+    model: Any,
+    *,
+    show_progress: bool = True,
+    **run_kwargs: Any,
+) -> list[OpenMLRun]:
+    """Run a model on all tasks in an OpenML suite with progress tracking.
+
+    Parameters
+    ----------
+    suite_id : int or str
+        OpenML suite ID or alias
+    model : estimator
+        A scikit-learn compatible estimator to benchmark
+    show_progress : bool, default=True
+        Whether to display a progress bar (requires tqdm)
+    **run_kwargs : dict
+        Additional keyword arguments passed to `openml.runs.run_model_on_task`
+
+    Returns
+    -------
+    list of OpenMLRun
+        List of successful run objects
+
+    Raises
+    ------
+    ImportError
+        If show_progress=True but tqdm is not installed
+
+    Examples
+    --------
+    >>> from sklearn.ensemble import RandomForestClassifier
+    >>> clf = RandomForestClassifier(n_estimators=10, random_state=42)
+    >>> runs = run_suite_with_progress(suite_id=99, model=clf)
+    >>> print(f"Completed {len(runs)} tasks")
+    """
+    if show_progress and not TQDM_AVAILABLE:
+        raise ImportError(
+            "tqdm is required for progress tracking. "
+            "Install it with: pip install tqdm\n"
+            "Or set show_progress=False"
+        )
+
+    # Get the suite
+    suite = openml.study.get_suite(suite_id)
+
+    if suite.tasks is None or len(suite.tasks) == 0:
+        return []
+
+    # Setup progress bar or plain iterator
+    if show_progress:
+        task_iterator = tqdm(
+            suite.tasks,
+            desc=f"Benchmarking {suite.name}",
+            unit="task",
+        )
+    else:
+        task_iterator = suite.tasks
+
+    # Run benchmark on each task
+    runs = []
+    for task_id in task_iterator:
+        result = openml.runs.run_model_on_task(model, task=task_id, **run_kwargs)
+        run = result[0] if isinstance(result, tuple) else result
+        runs.append(run)
+
+    return runs

openml/study/functions.py

@@ -1,5 +1,4 @@
 # License: BSD 3-Clause
-# ruff: noqa: PLR0913
 from __future__ import annotations
 
 import warnings
@@ -422,7 +421,7 @@ def detach_from_study(study_id: int, run_ids: list[int]) -> int:
         new size of the study (in terms of explicitly linked entities)
     """
     # Interestingly, there's no need to tell the server about the entity type, it knows by itself
-    uri = "study/%d/detach" % study_id
+    uri = f"study/{study_id}/detach"
     post_variables = {"ids": ",".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE
     result_xml = openml._api_calls._perform_api_call(
         call=uri,

tests/test_study/test_benchmarking.py

@@ -0,0 +1,57 @@
+# License: BSD 3-Clause
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openml.study.benchmarking import run_suite_with_progress
+from openml.testing import TestBase
+
+
+class TestRunSuiteWithProgress(TestBase):
+    """Tests for run_suite_with_progress function."""
+
+    def test_import_error_without_tqdm(self):
+        """Test that ImportError is raised when tqdm is unavailable."""
+        with patch("openml.study.benchmarking.TQDM_AVAILABLE", new=False):
+            with pytest.raises(ImportError, match="tqdm is required"):
+                run_suite_with_progress(
+                    suite_id=99,
+                    model=MagicMock(),
+                    show_progress=True,
+                )
+
+    @patch("openml.study.get_suite")
+    @patch("openml.runs.run_model_on_task")
+    def test_basic_functionality(self, mock_run_task, mock_get_suite):
+        """Test basic benchmarking with mocked suite."""
+        # Setup mock suite
+        mock_suite = MagicMock()
+        mock_suite.study_id = 99
+        mock_suite.name = "Test Suite"
+        mock_suite.tasks = [1, 2, 3]
+        mock_get_suite.return_value = mock_suite
+
+        # Setup mock runs
+        mock_runs = [MagicMock(run_id=100), MagicMock(run_id=200), MagicMock(run_id=300)]
+        mock_run_task.side_effect = mock_runs
+
+        # Run benchmark without progress bar
+        model = MagicMock()
+        runs = run_suite_with_progress(suite_id=99, model=model, show_progress=False)
+
+        # Assertions
+        assert len(runs) == 3
+        assert mock_run_task.call_count == 3
+
+    @patch("openml.study.get_suite")
+    def test_empty_suite(self, mock_get_suite):
+        """Test handling of empty suite."""
+        mock_suite = MagicMock()
+        mock_suite.tasks = []
+        mock_get_suite.return_value = mock_suite
+
+        runs = run_suite_with_progress(suite_id=99, model=MagicMock(), show_progress=False)
+
+        assert len(runs) == 0