add quick start risk control

Author: allglc

doc/index.rst

@@ -24,6 +24,7 @@
    :caption: Control prediction errors
 
    theoretical_description_risk_control
+   examples_risk_control/1-quickstart/plot_risk_control_binary_classification
    examples_risk_control/index
    external_risk_control_package
 

doc/quick_start.rst

@@ -40,4 +40,10 @@ Here, we generate one-dimensional noisy data that we fit with a MLPRegressor: `U
 3. Classification
 =======================
 
-Similarly, it's possible to do the same for a basic classification problem: `Use MAPIE to plot prediction sets <https://mapie.readthedocs.io/en/stable/examples_classification/1-quickstart/plot_quickstart_classification.html>`_
+Similarly, it's possible to do the same for a basic classification problem: `Use MAPIE to plot prediction sets <https://mapie.readthedocs.io/en/stable/examples_classification/1-quickstart/plot_quickstart_classification.html>`_
+
+
+4. Risk Control
+=======================
+
+MAPIE implements risk control methods for multilabel classification (in particular, image segmentation) and binary classification: `Use MAPIE to control risk for a binary classifier <https://mapie.readthedocs.io/en/stable/examples_risk_control/1-quickstart/plot_risk_control_binary_classification.html>`_

examples/risk_control/1-quickstart/plot_risk_control_binary_classification.py

@@ -0,0 +1,94 @@
+"""
+=================================================
+Use MAPIE to control risk for a binary classifier
+=================================================
+
+In this example, we explain how to do risk control for binary classification with MAPIE.
+
+"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.datasets import make_circles
+from sklearn.svm import SVC
+from sklearn.model_selection import FixedThresholdClassifier
+from sklearn.metrics import precision_score
+from sklearn.inspection import DecisionBoundaryDisplay
+
+from mapie.risk_control import BinaryClassificationController, precision
+from mapie.utils import train_conformalize_test_split
+
+RANDOM_STATE = 1
+
+##############################################################################
+# Let us first load the dataset and fit an SVC on the training data.
+
+X, y = make_circles(n_samples=3000, noise=0.3, factor=0.3, random_state=RANDOM_STATE)
+(X_train, X_calib, X_test, 
+ y_train, y_calib, y_test) = train_conformalize_test_split(
+     X, y, train_size=0.8, conformalize_size=0.1, test_size=0.1,
+     random_state=RANDOM_STATE)
+
+clf = SVC(probability=True, random_state=RANDOM_STATE)
+clf.fit(X_train, y_train)
+
+##############################################################################
+# Next, we initialize a :class:`~mapie.risk_control.BinaryClassificationController` using the probability estimation function from the fitted estimator: ``clf.predict_proba``, a risk function (here the precision), a target risk level, and a confidence level. Then we use the calibration data to compute statistically guaranteed thresholds using a risk control method.
+
+target_precision = 0.8
+bcc = BinaryClassificationController(
+    clf.predict_proba, precision, target_level=target_precision, confidence_level=0.9)
+bcc.calibrate(X_calib, y_calib)
+
+print(f'{len(bcc.valid_predict_params)} valid thresholds found. The best one is {bcc.best_predict_param:.3f}.')
+
+
+##############################################################################
+# In the plot below, we visualize how the threshold values impact precision, and what thresholds have been computed as statistically guaranteed.
+
+proba_positive_class = clf.predict_proba(X_calib)[:, 1]
+
+tested_thresholds = bcc._predict_params
+precisions = np.full(len(tested_thresholds), np.inf)
+for i, threshold in enumerate(tested_thresholds):
+    y_pred = (proba_positive_class >= threshold).astype(int)
+    precisions[i] = precision_score(y_calib, y_pred)
+
+valid_thresholds_indices = np.array([t in bcc.valid_predict_params for t in tested_thresholds])
+best_threshold_index = np.where(tested_thresholds == bcc.best_predict_param)[0][0]
+
+plt.figure()
+plt.scatter(tested_thresholds[valid_thresholds_indices], precisions[valid_thresholds_indices], c='tab:green', label='Valid thresholds')
+plt.scatter(tested_thresholds[~valid_thresholds_indices], precisions[~valid_thresholds_indices], c='tab:red', label='Invalid thresholds')
+plt.scatter(tested_thresholds[best_threshold_index], precisions[best_threshold_index], c='tab:green', label='Best threshold', marker='*', edgecolors='k', s=300)
+plt.axhline(target_precision, color='tab:gray', linestyle='--')
+plt.text(0, target_precision+0.02, 'Target precision', color='tab:gray', fontstyle='italic')
+plt.xlabel('Threshold', labelpad=15)
+plt.ylabel('Precision')
+plt.legend()
+plt.show()
+
+##############################################################################
+# Contrary to the naive way of computing a threshold to satisfy a precision target on calibration data, risk control provides statistical guarantees on unseen data. Besides computing a set of valid thresholds, :class:`~mapie.risk_control.BinaryClassificationController` also outputs the best one, which in the case of precision is the threshold that, among all valid ones, maximizes recall.
+# 
+# In the figure above, the highest threshold values are considered invalid due to the small number of observations used to compute the precision, following the Learn then Test procedure. In the most extreme case, no observation is available, which causes the precision value to be ill-defined and set to 0.
+#
+# After obtaining the best threshold, we can use the ``predict`` function of :class:`~mapie.risk_control.BinaryClassificationController` for future predictions, or use scikit-learn's ``FixedThresholdClassifier`` as a wrapper to benefit from functionalities like easily plotting the decision boundary as seen below.
+
+y_pred = bcc.predict(X_test)
+
+clf_threshold = FixedThresholdClassifier(clf, threshold=bcc.best_predict_param)
+clf_threshold.fit(X_train, y_train) # necessary for plotting, alternatively you can use sklearn.frozen.FrozenEstimator
+
+disp = DecisionBoundaryDisplay.from_estimator(clf_threshold, X_test, response_method="predict", cmap=plt.cm.coolwarm)
+
+plt.scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], edgecolors='k', c='tab:blue', alpha=0.5, label='"negative" class')
+plt.scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], edgecolors='k', c='tab:red', alpha=0.5, label='"positive" class')
+plt.title("Decision Boundary of FixedThresholdClassifier")
+plt.xlabel("Feature 1")
+plt.ylabel("Feature 2")
+plt.legend()
+plt.show()
+
+##############################################################################
+# Different risk functions have been implemented, such as precision and recall, but you can also implement your own custom function using :class:`~mapie.risk_control.BinaryClassificationRisk`.