ENH: replace regression_coverage_score by regression_coverage_score_v2, clarify docstring (#639)

---------

Co-authored-by: FaustinPulveric <[email protected]>

Author: Valentin-Laurent

doc/api.rst

@@ -92,7 +92,6 @@ Regression Metrics
    :toctree: generated/
    :template: function.rst
 
-   mapie.metrics.regression.regression_coverage_score
    mapie.metrics.regression.regression_coverage_score_v2
    mapie.metrics.regression.regression_mean_width_score
    mapie.metrics.regression.regression_ssc

examples/regression/1-quickstart/plot_compare_conformity_scores.py

@@ -42,7 +42,7 @@
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.model_selection import train_test_split
 
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import CrossConformalRegressor
 
 RANDOM_STATE = 42
@@ -111,9 +111,9 @@
     X_test
 )
 
-coverage_absconfscore = regression_coverage_score(
-    y_test, y_pis_absconfscore[:, 0, 0], y_pis_absconfscore[:, 1, 0]
-)
+coverage_absconfscore = regression_coverage_score_v2(
+    y_test, y_pis_absconfscore
+)[0]
 
 ##############################################################################
 # Prepare the results for matplotlib. Get the prediction intervals and their
@@ -146,9 +146,9 @@ def get_yerr(y_pred, y_pis):
     X_test
 )
 
-coverage_gammaconfscore = regression_coverage_score(
-    y_test, y_pis_gammaconfscore[:, 0, 0], y_pis_gammaconfscore[:, 1, 0]
-)
+coverage_gammaconfscore = regression_coverage_score_v2(
+    y_test, y_pis_gammaconfscore
+)[0]
 
 yerr_gammaconfscore = get_yerr(y_pred_gammaconfscore, y_pis_gammaconfscore)
 pred_int_width_gammaconfscore = (

examples/regression/1-quickstart/plot_prefit.py

@@ -27,7 +27,7 @@
 from sklearn.neural_network import MLPRegressor
 
 from numpy.typing import NDArray
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import SplitConformalRegressor, ConformalizedQuantileRegressor
 from mapie_v1.utils import train_conformalize_test_split
 
@@ -99,7 +99,7 @@ def f(x: NDArray) -> NDArray:
 
 # Evaluate prediction and coverage level on testing set
 y_pred, y_pis = mapie.predict_interval(X_test.reshape(-1, 1))
-coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1, 0])
+coverage = regression_coverage_score_v2(y_test, y_pis)[0]
 
 
 ##############################################################################
@@ -206,11 +206,10 @@ def f(x: NDArray) -> NDArray:
 
 # Evaluate prediction and coverage level on testing set
 y_pred_cqr, y_pis_cqr = mapie_cqr.predict_interval(X_test.reshape(-1, 1))
-coverage_cqr = regression_coverage_score(
+coverage_cqr = regression_coverage_score_v2(
     y_test,
-    y_pis_cqr[:, 0, 0],
-    y_pis_cqr[:, 1, 0]
-)
+    y_pis_cqr
+)[0]
 
 
 ##############################################################################

examples/regression/1-quickstart/plot_toy_model.py

@@ -9,7 +9,7 @@
 from matplotlib import pyplot as plt
 from sklearn.datasets import make_regression
 
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import SplitConformalRegressor
 from mapie_v1.utils import train_conformalize_test_split
 
@@ -33,11 +33,7 @@
 mapie_regressor.conformalize(X_conformalize, y_conformalize)
 y_pred, y_pred_interval = mapie_regressor.predict_interval(X_test)
 
-coverage_scores = [
-    regression_coverage_score(
-        y_test, y_pred_interval[:, 0, i], y_pred_interval[:, 1, i]
-    ) for i, _ in enumerate(confidence_level)
-]
+coverage_scores = regression_coverage_score_v2(y_test, y_pred_interval)
 
 plt.xlabel("x")
 plt.ylabel("y")

examples/regression/1-quickstart/plot_ts-tutorial.py

@@ -57,7 +57,7 @@ class that block bootstraps the training set.
 from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit
 
 from mapie.metrics.regression import (
-    regression_coverage_score,
+    regression_coverage_score_v2,
     regression_mean_width_score, coverage_width_based,
 )
 from mapie.regression import MapieTimeSeriesRegressor
@@ -217,9 +217,9 @@ class that block bootstraps the training set.
     allow_infinite_bounds=True
 )
 y_pis_enbpi_npfit = np.clip(y_pis_enbpi_npfit, 1, 10)
-coverage_enbpi_npfit = regression_coverage_score(
-    y_test, y_pis_enbpi_npfit[:, 0, 0], y_pis_enbpi_npfit[:, 1, 0]
-)
+coverage_enbpi_npfit = regression_coverage_score_v2(
+    y_test, y_pis_enbpi_npfit
+)[0]
 width_enbpi_npfit = regression_mean_width_score(
     y_pis_enbpi_npfit[:, 0, 0], y_pis_enbpi_npfit[:, 1, 0]
 )
@@ -258,9 +258,9 @@ class that block bootstraps the training set.
         y_pis_aci_npfit[step:step + gap, :, :], 1, 10
     )
 
-coverage_aci_npfit = regression_coverage_score(
-    y_test, y_pis_aci_npfit[:, 0, 0], y_pis_aci_npfit[:, 1, 0]
-)
+coverage_aci_npfit = regression_coverage_score_v2(
+    y_test, y_pis_aci_npfit
+)[0]
 width_aci_npfit = regression_mean_width_score(
     y_pis_aci_npfit[:, 0, 0], y_pis_aci_npfit[:, 1, 0]
 )
@@ -307,9 +307,9 @@ class that block bootstraps the training set.
     y_pis_enbpi_pfit[step:step + gap, :, :] = np.clip(
         y_pis_enbpi_pfit[step:step + gap, :, :], 1, 10
     )
-coverage_enbpi_pfit = regression_coverage_score(
-    y_test, y_pis_enbpi_pfit[:, 0, 0], y_pis_enbpi_pfit[:, 1, 0]
-)
+coverage_enbpi_pfit = regression_coverage_score_v2(
+    y_test, y_pis_enbpi_pfit
+)[0]
 width_enbpi_pfit = regression_mean_width_score(
     y_pis_enbpi_pfit[:, 0, 0], y_pis_enbpi_pfit[:, 1, 0]
 )
@@ -360,9 +360,9 @@ class that block bootstraps the training set.
         y_pis_aci_pfit[step:step + gap, :, :], 1, 10
     )
 
-coverage_aci_pfit = regression_coverage_score(
-    y_test, y_pis_aci_pfit[:, 0, 0], y_pis_aci_pfit[:, 1, 0]
-)
+coverage_aci_pfit = regression_coverage_score_v2(
+    y_test, y_pis_aci_pfit
+)[0]
 width_aci_pfit = regression_mean_width_score(
     y_pis_aci_pfit[:, 0, 0], y_pis_aci_pfit[:, 1, 0]
 )
@@ -462,29 +462,25 @@ class that block bootstraps the training set.
 
 for i in range(window, len(y_test), 1):
     rolling_coverage_aci_npfit.append(
-        regression_coverage_score(
-            y_test[i-window:i], y_pis_aci_npfit[i-window:i, 0, 0],
-            y_pis_aci_npfit[i-window:i, 1, 0]
-        )
+        regression_coverage_score_v2(
+            y_test[i-window:i], y_pis_aci_npfit[i-window:i]
+        )[0]
     )
     rolling_coverage_aci_pfit.append(
-        regression_coverage_score(
-            y_test[i-window:i], y_pis_aci_pfit[i-window:i, 0, 0],
-            y_pis_aci_pfit[i-window:i, 1, 0]
-        )
+        regression_coverage_score_v2(
+            y_test[i-window:i], y_pis_aci_pfit[i-window:i]
+        )[0]
     )
 
     rolling_coverage_enbpi_npfit.append(
-        regression_coverage_score(
-            y_test[i-window:i], y_pis_enbpi_npfit[i-window:i, 0, 0],
-            y_pis_enbpi_npfit[i-window:i, 1, 0]
-        )
+        regression_coverage_score_v2(
+            y_test[i-window:i], y_pis_enbpi_npfit[i-window:i]
+        )[0]
     )
     rolling_coverage_enbpi_pfit.append(
-        regression_coverage_score(
-            y_test[i-window:i], y_pis_enbpi_pfit[i-window:i, 0, 0],
-            y_pis_enbpi_pfit[i-window:i, 1, 0]
-        )
+        regression_coverage_score_v2(
+            y_test[i-window:i], y_pis_enbpi_pfit[i-window:i]
+        )[0]
     )
 
 plt.figure(figsize=(10, 5))

examples/regression/2-advanced-analysis/plot-coverage-width-based-criterion.py

@@ -25,7 +25,7 @@
 from sklearn.model_selection import train_test_split
 
 from mapie.metrics.regression import (
-    regression_coverage_score,
+    regression_coverage_score_v2,
     regression_mean_width_score, coverage_width_based,
 )
 from mapie_v1.regression import (
@@ -260,11 +260,10 @@ def plot_1d_data(
 cwc_score = {}
 
 for strategy in STRATEGIES:
-    coverage_score[strategy] = regression_coverage_score(
+    coverage_score[strategy] = regression_coverage_score_v2(
         y_test,
-        y_pis[strategy][:, 0, 0],
-        y_pis[strategy][:, 1, 0]
-    )
+        y_pis[strategy]
+    )[0]
     width_mean_score[strategy] = regression_mean_width_score(
         y_pis[strategy][:, 0, 0],
         y_pis[strategy][:, 1, 0]

examples/regression/2-advanced-analysis/plot_cqr_symmetry_difference.py

@@ -12,7 +12,7 @@
 from sklearn.datasets import make_regression
 from sklearn.ensemble import GradientBoostingRegressor
 
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import ConformalizedQuantileRegressor
 from mapie_v1.utils import train_conformalize_test_split
 
@@ -56,12 +56,12 @@
 print(f"y_pis_sym[:, 0].shape: {y_pis_sym[:, 0].shape}")
 print(f"y_pis_sym[:, 1].shape: {y_pis_sym[:, 1].shape}")
 # Calculate coverage scores
-coverage_score_sym = regression_coverage_score(
-    y_test, y_pis_sym[:, 0], y_pis_sym[:, 1]
-)
-coverage_score_asym = regression_coverage_score(
-    y_test, y_pis_asym[:, 0], y_pis_asym[:, 1]
-)
+coverage_score_sym = regression_coverage_score_v2(
+    y_test, y_pis_sym
+)[0]
+coverage_score_asym = regression_coverage_score_v2(
+    y_test, y_pis_asym
+)[0]
 
 # Sort the values for plotting
 order = np.argsort(X_test[:, 0])

examples/regression/2-advanced-analysis/plot_cqr_tutorial.py

@@ -30,7 +30,7 @@
 from sklearn.model_selection import KFold, RandomizedSearchCV, train_test_split
 
 from mapie.metrics.regression import (
-    regression_coverage_score,
+    regression_coverage_score_v2,
     regression_mean_width_score,
 )
 from mapie_v1.regression import (
@@ -277,11 +277,10 @@ def plot_prediction_intervals(
         lower_bound[strategy_name],
         upper_bound[strategy_name]
     ) = sort_y_values(y_test, y_pred[strategy_name], y_pis[strategy_name])
-    coverage[strategy_name] = regression_coverage_score(
+    coverage[strategy_name] = regression_coverage_score_v2(
         y_test,
-        y_pis[strategy_name][:, 0, 0],
-        y_pis[strategy_name][:, 1, 0]
-        )
+        y_pis[strategy_name]
+        )[0]
     width[strategy_name] = regression_mean_width_score(
         y_pis[strategy_name][:, 0, 0],
         y_pis[strategy_name][:, 1, 0]
@@ -363,11 +362,9 @@ def get_coverages_widths_by_bins(
             y_low_ = np.take(lower_bound[strategy], indices)
             y_high_ = np.take(upper_bound[strategy], indices)
             if want == "coverage":
-                recap[name].append(regression_coverage_score(
-                    y_test_trunc[0],
-                    y_low_[0],
-                    y_high_[0]
-                ))
+                recap[name].append(regression_coverage_score_v2(
+                    y_test_trunc[0], np.stack((y_low_[0], y_high_[0]), axis=-1)
+                )[0])
             elif want == "width":
                 recap[name].append(
                     regression_mean_width_score(y_low_[0], y_high_[0])

examples/regression/2-advanced-analysis/plot_main-tutorial-regression.py

@@ -37,7 +37,7 @@
 from sklearn.preprocessing import PolynomialFeatures
 from sklearn.model_selection import train_test_split
 
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import (
     CrossConformalRegressor,
     JackknifeAfterBootstrapRegressor,
@@ -303,9 +303,9 @@ def plot_1d_data(
 
 pd.DataFrame([
     [
-        regression_coverage_score(
-            y_test, y_pis[strategy][:, 0, 0], y_pis[strategy][:, 1, 0]
-        ),
+        regression_coverage_score_v2(
+            y_test, y_pis[strategy]
+        )[0],
         (
             y_pis[strategy][:, 1, 0] - y_pis[strategy][:, 0, 0]
         ).mean()
@@ -538,9 +538,9 @@ def get_heteroscedastic_coverage(y_test, y_pis, STRATEGIES, bins):
             y_test_trunc = np.take(y_test, indices)
             y_low_ = np.take(y_pis[strategy][:, 0, 0], indices)
             y_high_ = np.take(y_pis[strategy][:, 1, 0], indices)
-            score_coverage = regression_coverage_score(
-                y_test_trunc[0], y_low_[0], y_high_[0]
-            )
+            score_coverage = regression_coverage_score_v2(
+                y_test_trunc[0], np.stack((y_low_[0], y_high_[0]), axis=-1)
+            )[0]
             recap[name].append(score_coverage)
     recap_df = pd.DataFrame(recap, index=STRATEGIES)
     return recap_df
@@ -569,9 +569,9 @@ def get_heteroscedastic_coverage(y_test, y_pis, STRATEGIES, bins):
 
 pd.DataFrame([
     [
-        regression_coverage_score(
-            y_test, y_pis[strategy][:, 0, 0], y_pis[strategy][:, 1, 0]
-        ),
+        regression_coverage_score_v2(
+            y_test, y_pis[strategy]
+        )[0],
         (
             y_pis[strategy][:, 1, 0] - y_pis[strategy][:, 0, 0]
         ).mean()
@@ -766,9 +766,9 @@ def get_1d_data_with_normal_distrib(funct, mu, sigma, n_samples, noise):
 
 pd.DataFrame([
     [
-        regression_coverage_score(
-            y_test, y_pis[strategy][:, 0, 0], y_pis[strategy][:, 1, 0]
-        ),
+        regression_coverage_score_v2(
+            y_test, y_pis[strategy]
+        )[0],
         (
             y_pis[strategy][:, 1, 0] - y_pis[strategy][:, 0, 0]
         ).mean()

examples/regression/2-advanced-analysis/plot_nested-cv.py

@@ -45,7 +45,7 @@
 from sklearn.model_selection import RandomizedSearchCV, train_test_split
 from sklearn.datasets import make_sparse_uncorrelated
 
-from mapie.metrics.regression import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie_v1.regression import CrossConformalRegressor
 
 
@@ -91,9 +91,9 @@
     X_test, aggregate_predictions='median'
 )
 widths_non_nested = y_pis_non_nested[:, 1, 0] - y_pis_non_nested[:, 0, 0]
-coverage_non_nested = regression_coverage_score(
-    y_test, y_pis_non_nested[:, 0, 0], y_pis_non_nested[:, 1, 0]
-)
+coverage_non_nested = regression_coverage_score_v2(
+    y_test, y_pis_non_nested
+)[0]
 score_non_nested = root_mean_squared_error(y_test, y_pred_non_nested)
 
 # Nested approach with the CV+ strategy using the Random Forest model.
@@ -117,9 +117,9 @@
     X_test, aggregate_predictions='median'
 )
 widths_nested = y_pis_nested[:, 1, 0] - y_pis_nested[:, 0, 0]
-coverage_nested = regression_coverage_score(
-    y_test, y_pis_nested[:, 0, 0], y_pis_nested[:, 1, 0]
-)
+coverage_nested = regression_coverage_score_v2(
+    y_test, y_pis_nested
+)[0]
 score_nested = root_mean_squared_error(y_test, y_pred_nested)
 
 # Print scores and effective coverages.