diff --git a/mapie/conformity_scores/classification.py b/mapie/conformity_scores/classification.py index f5cbb10dd..6a882f781 100644 --- a/mapie/conformity_scores/classification.py +++ b/mapie/conformity_scores/classification.py @@ -2,11 +2,10 @@ from typing import Optional, Union import numpy as np - -from mapie.conformity_scores.interface import BaseConformityScore +from numpy.typing import ArrayLike, NDArray from sklearn.model_selection import BaseCrossValidator -from numpy.typing import ArrayLike, NDArray +from mapie.conformity_scores.interface import BaseConformityScore class BaseClassificationScore(BaseConformityScore, metaclass=ABCMeta): @@ -17,6 +16,12 @@ class BaseClassificationScore(BaseConformityScore, metaclass=ABCMeta): Attributes ---------- + classes: Optional[ArrayLike] + Names of the classes. + + random_state: Optional[Union[int, np.random.RandomState]] + Pseudo random number generator state. + quantiles_: ArrayLike of shape (n_alpha) The quantiles estimated from ``get_sets`` method. """ @@ -41,7 +46,7 @@ def set_external_attributes( By default ``None``. - random_state: Optional[Union[int, RandomState]] + random_state: Optional[Union[int, np.random.RandomState]] Pseudo random number generator state. """ super().set_external_attributes(**kwargs) @@ -71,8 +76,11 @@ def get_predictions( NDArray of floats between ``0`` and ``1``, represents the uncertainty of the confidence set. - estimator: EnsembleClassifier - Estimator that is fitted to predict y from X. + y_pred_proba: NDArray + Predicted probabilities from the estimator. + + cv: Optional[Union[int, str, BaseCrossValidator]] + Cross-validation strategy used by the estimator. Returns -------- @@ -102,8 +110,8 @@ def get_conformity_score_quantiles( NDArray of floats between 0 and 1, representing the uncertainty of the confidence set. - estimator: EnsembleClassifier - Estimator that is fitted to predict y from X. + cv: Optional[Union[int, str, BaseCrossValidator]] + Cross-validation strategy used by the estimator. Returns -------- @@ -138,8 +146,8 @@ def get_prediction_sets( NDArray of floats between 0 and 1, representing the uncertainty of the confidence set. - estimator: EnsembleClassifier - Estimator that is fitted to predict y from X. + cv: Optional[Union[int, str, BaseCrossValidator]] + Cross-validation strategy used by the estimator. Returns -------- @@ -211,12 +219,6 @@ def predict_set(self, X: NDArray, alpha_np: NDArray, **kwargs): alpha_np: NDArray of shape (n_alpha, ) Represents the uncertainty of the confidence set to produce. - y_pred_proba: NDArray - Predicted probabilities from the estimator. - - cv: Optional[Union[int, str, BaseCrossValidator]] - Cross-validation strategy used by the estimator. - **kwargs: dict Additional keyword arguments. diff --git a/mapie/conformity_scores/sets/aps.py b/mapie/conformity_scores/sets/aps.py index a11bc16ce..5304e7a61 100644 --- a/mapie/conformity_scores/sets/aps.py +++ b/mapie/conformity_scores/sets/aps.py @@ -1,15 +1,14 @@ from typing import Optional, Tuple, Union, cast import numpy as np -from sklearn.utils import check_random_state -from sklearn.preprocessing import label_binarize +from numpy.typing import ArrayLike, NDArray from sklearn.model_selection import BaseCrossValidator +from sklearn.preprocessing import label_binarize +from sklearn.utils import check_random_state +from mapie._machine_precision import EPSILON from mapie.conformity_scores.sets.naive import NaiveConformityScore from mapie.conformity_scores.sets.utils import check_include_last_label - -from mapie._machine_precision import EPSILON -from numpy.typing import ArrayLike, NDArray from mapie.utils import _compute_quantiles @@ -30,7 +29,7 @@ class APSConformityScore(NaiveConformityScore): classes: Optional[ArrayLike] Names of the classes. - random_state: Optional[Union[int, RandomState]] + random_state: Optional[Union[int, np.random.RandomState]] Pseudo random number generator state. quantiles_: ArrayLike of shape (n_alpha) @@ -65,7 +64,7 @@ def get_predictions( Predicted probabilities from the estimator. cv: Optional[Union[int, str, BaseCrossValidator]] - Cross-validation strategy used by the estimator. + Cross-validation strategy used by the estimator (not used here). agg_scores: Optional[str] Method to aggregate the scores from the base estimators. @@ -94,20 +93,20 @@ def get_true_label_cumsum_proba( Parameters ---------- - y: NDArray of shape (n_samples, ) + y: ArrayLike of shape (n_samples, ) Array with the labels. y_pred_proba: NDArray of shape (n_samples, n_classes) Predictions of the model. - classes: NDArray of shape (n_classes, ) + classes: ArrayLike of shape (n_classes, ) Array with the classes. Returns ------- Tuple[NDArray, NDArray] of shapes (n_samples, 1) and (n_samples, ). The first element is the cumsum probability of the true label. - The second is the sorted position of the true label. + The second is the 1-based rank of the true label in the sorted probabilities. """ y_true = label_binarize(y=y, classes=classes) index_sorted = np.fliplr(np.argsort(y_pred_proba, axis=1)) @@ -136,7 +135,7 @@ def get_conformity_scores( y_pred: NDArray of shape (n_samples,) Predicted target values. - y_enc: NDArray of shape (n_samples,) + y_enc: Optional[NDArray] of shape (n_samples,) Target values as normalized encodings. Returns @@ -225,8 +224,8 @@ def _compute_v_parameter( y_pred_proba_last: NDArray of shape (n_samples, 1, n_alpha) Last included probability. - predicition_sets: NDArray of shape (n_samples, n_alpha) - Prediction sets. + prediction_sets: NDArray of shape (n_samples, n_alpha) + Prediction sets (not used here). Returns -------- @@ -328,7 +327,7 @@ def get_prediction_sets( alpha_np: NDArray of shape (n_alpha,) NDArray of floats between 0 and 1, representing the uncertainty - of the confidence interval. + of the confidence interval (not used here). cv: Optional[Union[int, str, BaseCrossValidator]] Cross-validation strategy used by the estimator. diff --git a/mapie/conformity_scores/sets/lac.py b/mapie/conformity_scores/sets/lac.py index c89f4c1ed..3094c81dd 100644 --- a/mapie/conformity_scores/sets/lac.py +++ b/mapie/conformity_scores/sets/lac.py @@ -29,7 +29,7 @@ class LACConformityScore(BaseClassificationScore): classes: Optional[ArrayLike] Names of the classes. - random_state: Optional[Union[int, RandomState]] + random_state: Optional[Union[int, np.random.RandomState]] Pseudo random number generator state. quantiles_: ArrayLike of shape (n_alpha) @@ -48,7 +48,7 @@ def get_conformity_scores( Parameters ---------- y: NDArray of shape (n_samples,) - Observed target values. + Observed target values (not used here). y_pred: NDArray of shape (n_samples,) Predicted target values. @@ -94,7 +94,7 @@ def get_predictions( Predicted probabilities from the estimator. cv: Optional[Union[int, str, BaseCrossValidator]] - Cross-validation strategy used by the estimator. + Cross-validation strategy used by the estimator (not used here). agg_scores: Optional[str] Method to aggregate the scores from the base estimators. diff --git a/mapie/conformity_scores/sets/naive.py b/mapie/conformity_scores/sets/naive.py index dba611e83..4152d3409 100644 --- a/mapie/conformity_scores/sets/naive.py +++ b/mapie/conformity_scores/sets/naive.py @@ -1,13 +1,12 @@ -from typing import Tuple, Union, Optional +from typing import Optional, Tuple, Union import numpy as np - -from mapie.conformity_scores.classification import BaseClassificationScore -from mapie.conformity_scores.sets.utils import get_last_index_included +from numpy.typing import NDArray from sklearn.model_selection import BaseCrossValidator from mapie._machine_precision import EPSILON -from numpy.typing import NDArray +from mapie.conformity_scores.classification import BaseClassificationScore +from mapie.conformity_scores.sets.utils import get_last_index_included class NaiveConformityScore(BaseClassificationScore): @@ -20,10 +19,10 @@ class NaiveConformityScore(BaseClassificationScore): classes: Optional[ArrayLike] Names of the classes. - random_state: Optional[Union[int, RandomState]] + random_state: Optional[Union[int, np.random.RandomState]] Pseudo random number generator state. - quantiles_: ArrayLike of shape (n_alpha) + quantiles_: ArrayLike of shape (n_alpha,) The quantiles estimated from ``get_sets`` method. """ @@ -37,7 +36,7 @@ def get_conformity_scores(self, y: NDArray, y_pred: NDArray, **kwargs) -> NDArra Parameters ---------- y: NDArray of shape (n_samples,) - Observed target values. + Observed target values (not used here). y_pred: NDArray of shape (n_samples,) Predicted target values. @@ -97,11 +96,11 @@ def get_conformity_score_quantiles( Parameters ----------- conformity_scores: NDArray of shape (n_samples,) - Conformity scores for each sample. + Conformity scores for each sample (not used here). alpha_np: NDArray of shape (n_alpha,) NDArray of floats between 0 and 1, representing the uncertainty - of the confidence interval. + of the confidence interval (not used here). cv: Optional[Union[int, str, BaseCrossValidator]] Cross-validation strategy used by the estimator (not used here). @@ -222,11 +221,11 @@ def get_prediction_sets( Target prediction. conformity_scores: NDArray of shape (n_samples,) - Conformity scores for each sample. + Conformity scores for each sample (not used here). alpha_np: NDArray of shape (n_alpha,) NDArray of floats between 0 and 1, representing the uncertainty - of the confidence interval. + of the confidence interval (not used here). cv: Optional[Union[int, str, BaseCrossValidator]] Cross-validation strategy used by the estimator (not used here). diff --git a/mapie/conformity_scores/sets/raps.py b/mapie/conformity_scores/sets/raps.py index a57135bbd..f8bdd35b9 100644 --- a/mapie/conformity_scores/sets/raps.py +++ b/mapie/conformity_scores/sets/raps.py @@ -1,16 +1,15 @@ from typing import Optional, Tuple, Union, cast import numpy as np +from numpy.typing import NDArray +from sklearn.model_selection import BaseCrossValidator, StratifiedShuffleSplit from sklearn.preprocessing import LabelEncoder -from sklearn.model_selection import StratifiedShuffleSplit, BaseCrossValidator from sklearn.utils import _safe_indexing from sklearn.utils.validation import _num_samples +from mapie._machine_precision import EPSILON from mapie.conformity_scores.sets.aps import APSConformityScore from mapie.conformity_scores.sets.utils import get_true_label_position - -from mapie._machine_precision import EPSILON -from numpy.typing import NDArray from mapie.metrics.classification import classification_mean_width_score from mapie.utils import _check_alpha_and_n_samples, _compute_quantiles @@ -41,7 +40,7 @@ class RAPSConformityScore(APSConformityScore): classes: ArrayLike Names of the classes. - random_state: Union[int, RandomState] + random_state: Union[int, np.random.RandomState] Pseudo random number generator state. quantiles_: ArrayLike of shape (n_alpha) @@ -100,19 +99,19 @@ def split_data( Parameters ---------- - X: ArrayLike + X: NDArray Observed values. - y: ArrayLike + y: NDArray Target values. - y_enc: ArrayLike + y_enc: NDArray Target values as normalized encodings. - sample_weight: Optional[ArrayLike] of shape (n_samples,) + sample_weight: Optional[NDArray] of shape (n_samples,) Non-null sample weights. - groups: Optional[ArrayLike] of shape (n_samples,) + groups: Optional[NDArray] of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. By default ``None``. @@ -171,7 +170,7 @@ def get_conformity_scores( y_pred: NDArray of shape (n_samples,) Predicted target values. - y_enc: NDArray of shape (n_samples,) + y_enc: Optional[NDArray] of shape (n_samples,) Target values as normalized encodings. Returns @@ -370,7 +369,7 @@ def get_conformity_score_quantiles( agg_scores: Optional[str] Method to aggregate the scores from the base estimators. If "mean", the scores are averaged. If "crossval", the scores are - obtained from cross-validation. + obtained from cross-validation (not used here). By default, ``"mean"``. @@ -384,28 +383,6 @@ def get_conformity_score_quantiles( :meth:`conformity_scores.sets.aps.APSConformityScore.get_prediction_sets` for more details. - X_raps: NDArray of shape (n_samples, n_features) - Observed feature values for the RAPS method (split data). - - By default, "None" but must be set to work. - - y_raps_no_enc: NDArray of shape (n_samples,) - Observed labels for the RAPS method (split data). - - By default, "None" but must be set to work. - - y_pred_proba_raps: NDArray of shape (n_samples, n_classes) - Predicted probabilities for the RAPS method (split data). - - By default, "None" but must be set to work. - - position_raps: NDArray of shape (n_samples,) - Position of the points in the split set for the RAPS method - (split data). These positions are returned by the function - ``get_true_label_position``. - - By default, "None" but must be set to work. - Returns -------- NDArray @@ -453,17 +430,17 @@ def _add_regularization( y_pred_proba_sorted_cumsum: NDArray of shape (n_samples, n_classes) The sorted cumulative sum of predicted probabilities. - lambda_: float + lambda_: Optional[float] The lambda value used in the paper [1]. By default, "None" but must be set to work. - k_star: int + k_star: Optional[int] The optimal value of k (called k_reg in the paper [1]). By default, "None" but must be set to work. - prediction_phase: bool, optional + prediction_phase: bool Whether the function is called during the prediction phase. If ``True``, the function will use the values of ``lambda_star`` and ``k_star`` of the object. @@ -515,7 +492,7 @@ def _compute_v_parameter( y_pred_proba_last: NDArray of shape (n_samples, 1, n_alpha) Last included probability. - predicition_sets: NDArray of shape (n_samples, n_alpha) + prediction_sets: NDArray of shape (n_samples, n_alpha) Prediction sets. Returns diff --git a/mapie/conformity_scores/sets/topk.py b/mapie/conformity_scores/sets/topk.py index d8a2cdc55..43d4332cc 100644 --- a/mapie/conformity_scores/sets/topk.py +++ b/mapie/conformity_scores/sets/topk.py @@ -1,13 +1,12 @@ -from typing import Optional, cast, Union +from typing import Optional, Union, cast import numpy as np - -from mapie.conformity_scores.classification import BaseClassificationScore -from mapie.conformity_scores.sets.utils import get_true_label_position +from numpy.typing import NDArray from sklearn.model_selection import BaseCrossValidator from mapie._machine_precision import EPSILON -from numpy.typing import NDArray +from mapie.conformity_scores.classification import BaseClassificationScore +from mapie.conformity_scores.sets.utils import get_true_label_position from mapie.utils import _compute_quantiles @@ -32,7 +31,7 @@ class TopKConformityScore(BaseClassificationScore): classes: Optional[ArrayLike] Names of the classes. - random_state: Optional[Union[int, RandomState]] + random_state: Optional[Union[int, np.random.RandomState]] Pseudo random number generator state. quantiles_: ArrayLike of shape (n_alpha) @@ -85,8 +84,6 @@ def get_predictions( """ Just processes the passed y_pred_proba. - This method should be implemented by any subclass of the current class. - Parameters ----------- X: NDArray of shape (n_samples, n_features) @@ -157,11 +154,11 @@ def get_prediction_sets( Target prediction. conformity_scores: NDArray of shape (n_samples,) - Conformity scores for each sample. + Conformity scores for each sample (not used here). alpha_np: NDArray of shape (n_alpha,) NDArray of floats between 0 and 1, representing the uncertainty - of the confidence interval. + of the confidence interval (not used here). cv: Optional[Union[int, str, BaseCrossValidator]] Cross-validation strategy used by the estimator (not used here).