MTN: split metrics into 3 modules (regression, classification, calibration). Update doc accordingly (#633)

Author: Valentin-Laurent

doc/api.rst

@@ -52,36 +52,54 @@ Calibration
 
    mapie.calibration.MapieCalibrator
 
-Metrics
-=======
+Calibration Metrics
+======================================
 
 .. autosummary::
    :toctree: generated/
    :template: function.rst
 
-   mapie.metrics.classification_coverage_score
-   mapie.metrics.classification_coverage_score_v2
-   mapie.metrics.classification_mean_width_score
-   mapie.metrics.classification_ssc
-   mapie.metrics.classification_ssc_score
-   mapie.metrics.cumulative_differences
-   mapie.metrics.expected_calibration_error
-   mapie.metrics.hsic
-   mapie.metrics.kolmogorov_smirnov_cdf
-   mapie.metrics.kolmogorov_smirnov_p_value
-   mapie.metrics.kolmogorov_smirnov_statistic
-   mapie.metrics.kuiper_cdf
-   mapie.metrics.kuiper_p_value
-   mapie.metrics.kuiper_statistic
-   mapie.metrics.length_scale
-   mapie.metrics.regression_coverage_score
-   mapie.metrics.regression_coverage_score_v2
-   mapie.metrics.regression_mean_width_score
-   mapie.metrics.regression_ssc
-   mapie.metrics.regression_ssc_score
-   mapie.metrics.spiegelhalter_p_value
-   mapie.metrics.spiegelhalter_statistic
-   mapie.metrics.top_label_ece
+   mapie.metrics.calibration.expected_calibration_error
+   mapie.metrics.calibration.top_label_ece
+   mapie.metrics.calibration.cumulative_differences
+   mapie.metrics.calibration.kolmogorov_smirnov_cdf
+   mapie.metrics.calibration.kolmogorov_smirnov_p_value
+   mapie.metrics.calibration.kolmogorov_smirnov_statistic
+   mapie.metrics.calibration.kuiper_cdf
+   mapie.metrics.calibration.kuiper_p_value
+   mapie.metrics.calibration.kuiper_statistic
+   mapie.metrics.calibration.length_scale
+   mapie.metrics.calibration.spiegelhalter_p_value
+   mapie.metrics.calibration.spiegelhalter_statistic
+
+Classification Metrics
+========================================
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+
+   mapie.metrics.classification.classification_coverage_score
+   mapie.metrics.classification.classification_coverage_score_v2
+   mapie.metrics.classification.classification_mean_width_score
+   mapie.metrics.classification.classification_ssc
+   mapie.metrics.classification.classification_ssc_score
+
+Regression Metrics
+====================================
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+
+   mapie.metrics.regression.regression_coverage_score
+   mapie.metrics.regression.regression_coverage_score_v2
+   mapie.metrics.regression.regression_mean_width_score
+   mapie.metrics.regression.regression_ssc
+   mapie.metrics.regression.regression_ssc_score
+   mapie.metrics.regression.hsic
+   mapie.metrics.regression.coverage_width_based
+   mapie.metrics.regression.regression_mwi_score
 
 Utils (from mapie_v1)
 ==============================

doc/quick_start.rst

@@ -69,7 +69,7 @@ Here, we generate one-dimensional noisy data that we fit with a linear model.
 
     # You can compute the coverage of your prediction intervals.
 
-    from mapie.metrics import regression_coverage_score_v2
+    from mapie.metrics.regression import regression_coverage_score_v2
 
     coverage_scores = regression_coverage_score_v2(y_test, y_pred_intervals)
 

examples/calibration/1-quickstart/plot_calibration_hypothesis_testing.py

@@ -30,8 +30,10 @@
 from sklearn.utils import check_random_state
 
 from numpy.typing import NDArray
-from mapie.metrics import (cumulative_differences, kolmogorov_smirnov_p_value,
-                           length_scale)
+from mapie.metrics.calibration import (
+    cumulative_differences, kolmogorov_smirnov_p_value,
+    length_scale,
+)
 
 ####################################################################
 # 1. Create 1-dimensional dataset and scores to test for calibration

examples/calibration/2-advanced-analysis/plot_asymptotic_convergence_of_p_values.py

@@ -40,8 +40,8 @@
 from sklearn.utils import check_random_state
 
 from numpy.typing import NDArray
-from mapie.metrics import (kolmogorov_smirnov_p_value, kuiper_p_value,
-                           spiegelhalter_p_value)
+from mapie.metrics.calibration import (spiegelhalter_p_value)
+from mapie.metrics.calibration import kolmogorov_smirnov_p_value, kuiper_p_value
 
 ##############################################################################
 # First we need to generate scores that are perfecty calibrated. To do so,

examples/classification/1-quickstart/plot_comp_methods_on_2d_dataset.py

@@ -55,8 +55,10 @@
 
 from numpy.typing import NDArray
 from mapie.classification import MapieClassifier
-from mapie.metrics import (classification_coverage_score,
-                           classification_mean_width_score)
+from mapie.metrics.classification import (
+    classification_coverage_score,
+    classification_mean_width_score,
+)
 
 centers = [(0, 3.5), (-2, 0), (2, 0)]
 covs = [np.eye(2), np.eye(2) * 2, np.diag([5, 1])]

examples/classification/4-tutorials/plot_crossconformal.py

@@ -37,8 +37,10 @@
 
 from numpy.typing import NDArray
 from mapie.classification import MapieClassifier
-from mapie.metrics import (classification_coverage_score,
-                           classification_mean_width_score)
+from mapie.metrics.classification import (
+    classification_coverage_score,
+    classification_mean_width_score,
+)
 
 ##############################################################################
 # 1. Estimating the impact of train/calibration split on the prediction sets

examples/classification/4-tutorials/plot_main-tutorial-binary-classification.py

@@ -35,8 +35,10 @@
 
 from numpy.typing import NDArray
 from mapie.classification import MapieClassifier
-from mapie.metrics import (classification_coverage_score,
-                           classification_mean_width_score)
+from mapie.metrics.classification import (
+    classification_coverage_score,
+    classification_mean_width_score,
+)
 
 ##############################################################################
 # 1. Conformal Prediction method using the softmax score of the true label

examples/classification/4-tutorials/plot_main-tutorial-classification.py

@@ -23,8 +23,10 @@
 from sklearn.naive_bayes import GaussianNB
 
 from mapie.classification import MapieClassifier
-from mapie.metrics import (classification_coverage_score,
-                           classification_mean_width_score)
+from mapie.metrics.classification import (
+    classification_coverage_score,
+    classification_mean_width_score,
+)
 
 ##############################################################################
 # 1. Conformal Prediction method using the softmax score of the true label

examples/mondrian/1-quickstart/plot_main-tutorial-mondrian-regression.py

@@ -26,7 +26,7 @@
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import RandomForestRegressor
 
-from mapie.metrics import regression_coverage_score_v2
+from mapie.metrics.regression import regression_coverage_score_v2
 from mapie.mondrian import MondrianCP
 from mapie.regression import MapieRegressor
 

examples/regression/1-quickstart/plot_compare_conformity_scores.py

@@ -42,7 +42,7 @@
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.model_selection import train_test_split
 
-from mapie.metrics import regression_coverage_score
+from mapie.metrics.regression import regression_coverage_score
 from mapie_v1.regression import CrossConformalRegressor
 
 RANDOM_STATE = 42