Fix the bug for calculating auc-pr/roc.

Author: lartpang

examples/metric_recorder.py

@@ -148,6 +148,10 @@ def get_results(self, num_bits: int = 3, return_ndarray: bool = False) -> dict:
 SIZEINVARIANCE_METRIC_MAPPING = {
     "handler":{
         "si_fm": {"handler": py_sod_metrics.FmeasureHandler, "kwargs": dict(**sample_gray, beta=0.3)},
+        "si_pre": {"handler": py_sod_metrics.PrecisionHandler, "kwargs": dict(with_adaptive=False, with_dynamic=True, sample_based=True)},
+        "si_rec": {"handler": py_sod_metrics.RecallHandler, "kwargs": dict(with_adaptive=False, with_dynamic=True, sample_based=True)},
+        "si_tpr": {"handler": py_sod_metrics.TPRHandler, "kwargs": dict(with_adaptive=False, with_dynamic=True, sample_based=True)},
+        "si_fpr": {"handler": py_sod_metrics.FPRHandler, "kwargs": dict(with_adaptive=False, with_dynamic=True, sample_based=True)},
     },
     "si_fmeasurev2": py_sod_metrics.SizeInvarianceFmeasureV2,
     "si_mae": py_sod_metrics.SizeInvarianceMAE,
@@ -319,13 +323,49 @@ def step(self, pre: np.ndarray, gt: np.ndarray):
         for m_obj in self.metric_objs.values():
             m_obj.step(pre, gt)
 
+    def cal_auc(self, y, x):
+        sorted_idx = np.argsort(x, axis=-1, kind="stable")
+        x = np.take_along_axis(x, sorted_idx, axis=-1)
+        y = np.take_along_axis(y, sorted_idx, axis=-1)
+        return np.trapz(y, x, axis=-1)
+
     def get_all_results(self, num_bits: int = 3, return_ndarray: bool = False) -> dict:
         sequential_results = {}
         numerical_results = {}
         for m_name, m_obj in self.metric_objs.items():
             info = m_obj.get_results()
 
             if m_name == "si_fmeasurev2":
+                # AUC-ROC
+                if "si_tpr" in info and "si_fpr" in info:
+                    ys = info.pop("si_tpr")["dynamic"]  # >=255,>=254,...>=1,>=0
+                    xs = info.pop("si_fpr")["dynamic"]
+                    if isinstance(ys, list) and isinstance(xs, list):  # Nx[T'x256]
+                        auc_results = []
+                        for y, x in zip(ys, xs):
+                            # NOTE: before calculate the auc, we need to flip the y and x to corresponding to ascending thresholds
+                            # because the dynamic results from our metrics is based on the descending order of thresholds, i.e., >=255,>=254,...>=1,>=0
+                            y = np.flip(y, axis=-1)
+                            x = np.flip(x, axis=-1)
+                            auc_results.append(self.cal_auc(y, x).mean())
+                        numerical_results["si_sample_auc_roc"] = np.asarray(auc_results).mean()
+                    else:  # 256
+                        numerical_results["si_overall_auc_roc"] = self.cal_auc(y=ys, x=xs).mean()
+
+                # AUC-PR
+                if "si_pre" in info and "si_rec" in info:
+                    ys = info.pop("si_pre")["dynamic"]  # >=255,>=254,...>=1,>=0
+                    xs = info.pop("si_rec")["dynamic"]
+                    if isinstance(ys, list) and isinstance(xs, list):  # Nx[T'x256]
+                        auc_results = []
+                        for y, x in zip(ys, xs):
+                            y = np.flip(y, axis=-1)
+                            x = np.flip(x, axis=-1)
+                            auc_results.append(self.cal_auc(y, x).mean())
+                        numerical_results["si_sample_auc_pr"] = np.asarray(auc_results).mean()
+                    else:  # 256
+                        numerical_results["si_overall_auc_pr"] = self.cal_auc(y=ys, x=xs).mean()
+
                 for _name, results in info.items():
                     dynamic_results = results.get("dynamic")
                     if dynamic_results is not None:
@@ -338,17 +378,17 @@ def get_all_results(self, num_bits: int = 3, return_ndarray: bool = False) -> di
                                 avg_results.append(s.mean(axis=-1).mean())  # 1
                                 seq_results.append(s.mean(axis=0))  # 256
                             seq_results = np.mean(np.asarray(seq_results), axis=0)
-                            numerical_results[f"max{_name}"] = np.asarray(max_results).mean()
-                            numerical_results[f"avg{_name}"] = np.asarray(avg_results).mean()
+                            numerical_results[f"si_sample_max{_name}"] = np.asarray(max_results).mean()
+                            numerical_results[f"si_sample_avg{_name}"] = np.asarray(avg_results).mean()
                         else:  # 256
                             seq_results = dynamic_results
-                            numerical_results[f"max{_name}"] = dynamic_results.max()
-                            numerical_results[f"avg{_name}"] = dynamic_results.mean()
+                            numerical_results[f"si_overall_max{_name}"] = dynamic_results.max()
+                            numerical_results[f"si_overall_avg{_name}"] = dynamic_results.mean()
                         sequential_results[_name] = np.flip(seq_results)
 
                     adaptive_results = results.get("adaptive")
                     if adaptive_results is not None:
-                        numerical_results[f"adp{_name}"] = adaptive_results
+                        numerical_results[f"si_sample_adp{_name}"] = adaptive_results
             else:
                 results = info[m_name]
                 if m_name in ("si_mae",):

examples/test_metrics.py

@@ -34,6 +34,10 @@ def reduce_dynamic_results_for_max_avg(dynamic_results: list):  # Nx[T'x256] ->
 def reduce_dynamic_results_for_auc(ys: list, xs: list):  # Nx[T'x256] -> Nx[T'] -> N -> 1
     auc_results = []
     for y, x in zip(ys, xs):
+        # NOTE: before calculate the auc, we need to flip the y and x to corresponding to ascending thresholds
+        # because the dynamic results from our metrics is based on the descending order of thresholds, i.e., >=255,>=254,...>=1,>=0
+        y = np.flip(y, -1)
+        x = np.flip(x, -1)
         auc_results.append(cal_auc(y=y, x=x).mean())
     return np.asarray(auc_results).mean()
 
@@ -230,14 +234,18 @@ def setUpClass(cls):
         pr_pre = fmv2["pre"]["dynamic"]  # 256
         pr_rec = fmv2["rec"]["dynamic"]  # 256
         roc_fpr = fmv2["fpr"]["dynamic"]  # tpr is the same as recall
-        cls.curr_results["auc_pr"] = cal_auc(y=pr_pre, x=pr_rec)
-        cls.curr_results["auc_roc"] = cal_auc(y=pr_rec, x=roc_fpr)
+        cls.curr_results["auc_pr"] = cal_auc(y=np.flip(pr_pre, -1), x=np.flip(pr_rec, -1))
+        cls.curr_results["auc_roc"] = cal_auc(y=np.flip(pr_rec, -1), x=np.flip(roc_fpr, -1))
 
         si_overall_pr_pre = si_fmv2["si_overall_pre"]["dynamic"]  # 256
         si_overall_pr_rec = si_fmv2["si_overall_rec"]["dynamic"]  # 256
         si_overall_roc_fpr = si_fmv2["si_overall_fpr"]["dynamic"]  # 256
-        cls.curr_results["si_overall_auc_pr"] = cal_auc(y=si_overall_pr_pre, x=si_overall_pr_rec)
-        cls.curr_results["si_overall_auc_roc"] = cal_auc(y=si_overall_pr_rec, x=si_overall_roc_fpr)
+        cls.curr_results["si_overall_auc_pr"] = cal_auc(
+            y=np.flip(si_overall_pr_pre, -1), x=np.flip(si_overall_pr_rec, -1)
+        )
+        cls.curr_results["si_overall_auc_roc"] = cal_auc(
+            y=np.flip(si_overall_pr_rec, -1), x=np.flip(si_overall_roc_fpr, -1)
+        )
 
         si_sample_pr_pre = si_fmv2["si_sample_pre"]["dynamic"]  # Nx[T'x256]
         si_sample_pr_rec = si_fmv2["si_sample_rec"]["dynamic"]  # Nx[T'x256]