test - handle RuntimeWarnings in unit tests (#802)

* capture and test expected RuntimeWarning

* capture and test MAPIE warnings since it is expected

* filter (ignore) RuntimeWarning because it is tested by another test

* rename  test_not_all_label_in_calib() into test_y_pred_shape_in_calib() for clarity

* simplify tests in test_classification.py

* debug test_sum_proba_to_one_fit : increase LogisticRegression max_iter and ravel simulated y

Author: hmaissoro

mapie/metrics/classification.py

@@ -1,14 +1,14 @@
-from typing import cast, Union
+from typing import Union, cast
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from sklearn.utils import column_or_1d
 
 from mapie.utils import (
-    _check_arrays_length,
-    _check_array_nan,
     _check_array_inf,
+    _check_array_nan,
     _check_array_shape_classification,
+    _check_arrays_length,
     _check_nb_sets_sizes,
     _check_number_bins,
 )

mapie/tests/risk_control/test_risk_control.py

@@ -66,6 +66,7 @@ def test_compute_precision() -> None:
     np.testing.assert_equal(precision, test_precision)
 
 
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_recall_with_zero_sum_is_equal_nan() -> None:
     """Test compute_recall with nan values"""
     y_toy = np.zeros((4, 3))

mapie/tests/test_classification.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 import pytest
+from numpy.typing import ArrayLike, NDArray
 from sklearn.base import ClassifierMixin
 from sklearn.compose import ColumnTransformer
 from sklearn.datasets import make_classification
@@ -25,18 +26,17 @@
 from sklearn.utils.validation import check_is_fitted
 from typing_extensions import TypedDict
 
-from numpy.typing import ArrayLike, NDArray
 from mapie.classification import _MapieClassifier
 from mapie.conformity_scores import (
-    LACConformityScore,
-    RAPSConformityScore,
     APSConformityScore,
     BaseClassificationScore,
-    TopKConformityScore,
+    LACConformityScore,
     NaiveConformityScore,
+    RAPSConformityScore,
+    TopKConformityScore,
 )
-from mapie.utils import check_proba_normalized
 from mapie.metrics.classification import classification_coverage_score
+from mapie.utils import check_proba_normalized
 
 random_state = 42
 
@@ -1407,6 +1407,7 @@ def test_image_cumulated_scores(X: Dict[str, ArrayLike]) -> None:
     np.testing.assert_allclose(y_ps[:, :, 0], cumclf.y_pred_sets)
 
 
+@pytest.mark.filterwarnings("ignore:: UserWarning")
 @pytest.mark.parametrize("y_pred_proba", Y_PRED_PROBA_WRONG)
 def test_sum_proba_to_one_fit(y_pred_proba: NDArray) -> None:
     """
@@ -1501,7 +1502,8 @@ def test_pipeline_compatibility(strategy: str) -> None:
             "x_num": X,
         }
     )
-    y = np.random.randint(0, 4, size=(100, 1))  # 3 classes
+    # 3 classes
+    y = np.random.randint(0, 4, size=(100, 1)).ravel()
     numeric_preprocessor = Pipeline(
         [
             ("imputer", SimpleImputer(strategy="mean")),
@@ -1516,7 +1518,7 @@ def test_pipeline_compatibility(strategy: str) -> None:
             ("num", numeric_preprocessor, ["x_num"]),
         ]
     )
-    pipe = make_pipeline(preprocessor, LogisticRegression())
+    pipe = make_pipeline(preprocessor, LogisticRegression(max_iter=500))
     pipe.fit(X, y)
     mapie = _MapieClassifier(estimator=pipe, **STRATEGIES[strategy][0])
     mapie.fit(X, y)
@@ -1578,23 +1580,20 @@ def test_error_raps_cv_not_prefit(cv: Union[int, None]) -> None:
         mapie.fit(X_toy, y_toy)
 
 
-def test_not_all_label_in_calib() -> None:
+def test_y_pred_shape_in_calib() -> None:
     """
     Test that the true label cumsumed probabilities
     have the correct shape.
     """
     clf = LogisticRegression()
     clf.fit(X, y)
-    indices_remove = np.where(y != 2)
-    X_mapie = X[indices_remove]
-    y_mapie = y[indices_remove]
     mapie_clf = _MapieClassifier(
         estimator=clf,
         conformity_score=APSConformityScore(),
         cv="prefit",
         random_state=random_state,
     )
-    mapie_clf.fit(X_mapie, y_mapie)
+    mapie_clf.fit(X, y)
     y_pred, y_pss = mapie_clf.predict(X, alpha=0.5)
     assert y_pred.shape == (len(X),)
     assert y_pss.shape == (len(X), len(np.unique(y)), 1)
@@ -1626,16 +1625,13 @@ def test_n_classes_prefit() -> None:
     """
     clf = LogisticRegression()
     clf.fit(X, y)
-    indices_remove = np.where(y != 2)
-    X_mapie = X[indices_remove]
-    y_mapie = y[indices_remove]
     mapie_clf = _MapieClassifier(
         estimator=clf,
         conformity_score=APSConformityScore(),
         cv="prefit",
         random_state=random_state,
     )
-    mapie_clf.fit(X_mapie, y_mapie)
+    mapie_clf.fit(X, y)
     assert mapie_clf.n_classes_ == len(np.unique(y))
 
 
@@ -1646,16 +1642,13 @@ def test_classes_prefit() -> None:
     """
     clf = LogisticRegression()
     clf.fit(X, y)
-    indices_remove = np.where(y != 2)
-    X_mapie = X[indices_remove]
-    y_mapie = y[indices_remove]
     mapie_clf = _MapieClassifier(
         estimator=clf,
         conformity_score=APSConformityScore(),
         cv="prefit",
         random_state=random_state,
     )
-    mapie_clf.fit(X_mapie, y_mapie)
+    mapie_clf.fit(X, y)
     assert (mapie_clf.classes_ == np.unique(y)).all()
 
 
@@ -1666,13 +1659,10 @@ def test_classes_encoder_same_than_model() -> None:
     """
     clf = LogisticRegression()
     clf.fit(X, y)
-    indices_remove = np.where(y != 2)
-    X_mapie = X[indices_remove]
-    y_mapie = y[indices_remove]
     mapie_clf = _MapieClassifier(
         estimator=clf, conformity_score=APSConformityScore(), cv="prefit"
     )
-    mapie_clf.fit(X_mapie, y_mapie)
+    mapie_clf.fit(X, y)
     assert (mapie_clf.label_encoder_.classes_ == np.unique(y)).all()
 
 

mapie/tests/test_metrics.py

@@ -7,10 +7,9 @@
 import numpy as np
 import pytest
 from numpy.random import RandomState
+from numpy.typing import ArrayLike, NDArray
 from typing_extensions import TypedDict
 
-from numpy.typing import ArrayLike, NDArray
-from mapie.metrics.calibration import spiegelhalter_p_value
 from mapie.metrics.calibration import (
     add_jitter,
     cumulative_differences,
@@ -23,23 +22,24 @@
     kuiper_statistic,
     length_scale,
     sort_xy_by_y,
+    spiegelhalter_p_value,
     spiegelhalter_statistic,
     top_label_ece,
 )
 from mapie.metrics.classification import (
-    classification_mean_width_score,
     classification_coverage_score,
+    classification_mean_width_score,
     classification_ssc,
     classification_ssc_score,
 )
 from mapie.metrics.regression import (
-    regression_mean_width_score,
+    coverage_width_based,
+    hsic,
     regression_coverage_score,
+    regression_mean_width_score,
+    regression_mwi_score,
     regression_ssc,
     regression_ssc_score,
-    hsic,
-    coverage_width_based,
-    regression_mwi_score,
 )
 
 y_toy = np.array([5, 7.5, 9.5, 10.5, 12.5])
@@ -263,8 +263,14 @@ def test_classification_same_length() -> None:
 
 def test_classification_valid_input_shape() -> None:
     """Test that valid inputs shape raise no error."""
-    classification_ssc(y_true_class, y_pred_set_2alphas)
-    classification_ssc_score(y_true_class, y_pred_set_2alphas)
+    with pytest.warns(
+        RuntimeWarning, match=r".*empty slice.*|.*invalid value encountered in divide.*"
+    ):
+        classification_ssc(y_true_class, y_pred_set_2alphas)
+    with pytest.warns(
+        RuntimeWarning, match=r".*empty slice.*|.*invalid value encountered in divide.*"
+    ):
+        classification_ssc_score(y_true_class, y_pred_set_2alphas)
     classification_mean_width_score(y_pred_set_2alphas)
 
 
@@ -410,39 +416,45 @@ def test_invalid_splits_classification_ssc_score(num_bins: int) -> None:
 
 
 @pytest.mark.parametrize("num_bins", [3, 2, None])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_valid_splits_classification_ssc(num_bins: int) -> None:
-    """Test that valid number of bins for ssc raise no error."""
+    """Test that valid number of bins for ssc raise no ValueError."""
     classification_ssc(y_true_class, y_pred_set_2alphas, num_bins=num_bins)
 
 
 @pytest.mark.parametrize("num_bins", [3, 2, None])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_valid_splits_classification_ssc_score(num_bins: int) -> None:
-    """Test that valid number of bins for ssc raise no error."""
+    """Test that valid number of bins for ssc raise no ValueError."""
     classification_ssc_score(y_true_class, y_pred_set_2alphas, num_bins=num_bins)
 
 
 @pytest.mark.parametrize("params", [*SSC_CLASSIF])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_classification_ssc_return_shape(params: str) -> None:
     """Test that the arrays returned by ssc metrics have the correct shape."""
     cond_cov = classification_ssc(y_true_class, **SSC_CLASSIF[params])
     assert cond_cov.shape == SSC_CLASSIF_COVERAGES[params].shape
 
 
 @pytest.mark.parametrize("params", [*SSC_CLASSIF])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_classification_ssc_score_return_shape(params: str) -> None:
     """Test that the arrays returned by ssc metrics have the correct shape."""
     cond_cov_min = classification_ssc_score(y_true_class, **SSC_CLASSIF[params])
     assert cond_cov_min.shape == SSC_CLASSIF_COVERAGES_SCORE[params].shape
 
 
 @pytest.mark.parametrize("params", [*SSC_CLASSIF])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_classification_ssc_coverage_values(params: str) -> None:
     """Test that the conditional coverage values returned are correct."""
     cond_cov = classification_ssc(y_true_class, **SSC_CLASSIF[params])
     np.testing.assert_allclose(cond_cov, SSC_CLASSIF_COVERAGES[params])
 
 
 @pytest.mark.parametrize("params", [*SSC_CLASSIF])
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_classification_ssc_score_coverage_values(params: str) -> None:
     """Test that the conditional coverage values returned are correct."""
     cond_cov_min = classification_ssc_score(y_true_class, **SSC_CLASSIF[params])

mapie/tests/test_regression.py

@@ -721,6 +721,7 @@ def test_results_prefit() -> None:
     np.testing.assert_allclose(coverage, COVERAGES["prefit"], rtol=1e-2)
 
 
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_not_enough_resamplings() -> None:
     """
     Test that a warning is raised if at least one conformity score is nan.

mapie/tests/test_time_series_regression.py

@@ -303,6 +303,7 @@ def test_results_prefit() -> None:
     np.testing.assert_allclose(coverage, COVERAGES["prefit"], rtol=1e-2)
 
 
+@pytest.mark.filterwarnings("ignore:: RuntimeWarning")
 def test_not_enough_resamplings() -> None:
     """Test that a warning is raised if at least one residual is nan."""
     with pytest.warns(UserWarning, match=r"WARNING: at least one point of*"):