Merge pull request #106 from ImageMarkup/customizable-validation-metric

Author: danlamanna

isic_challenge_scoring/__init__.py

@@ -1,5 +1,5 @@
-from isic_challenge_scoring.classification import ClassificationScore
+from isic_challenge_scoring.classification import ClassificationScore, ValidationMetric
 from isic_challenge_scoring.segmentation import SegmentationScore
 from isic_challenge_scoring.types import ScoreException
 
-__all__ = ['ClassificationScore', 'SegmentationScore', 'ScoreException']
+__all__ = ['ClassificationScore', 'SegmentationScore', 'ScoreException', 'ValidationMetric']

isic_challenge_scoring/classification.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+import enum
 import pathlib
-from typing import Dict, TextIO, cast
+from typing import Dict, Optional, TextIO, cast
 
 import pandas as pd
 
@@ -12,6 +13,12 @@
 from isic_challenge_scoring.types import DataFrameDict, RocDict, Score, ScoreDict, SeriesDict
 
 
+class ValidationMetric(enum.Enum):
+    BALANCED_ACCURACY = 'balanced_accuracy'
+    AUC = 'auc'
+    AVERAGE_PRECISION = 'ap'
+
+
 @dataclass(init=False)
 class ClassificationScore(Score):
     per_category: pd.DataFrame
@@ -24,6 +31,7 @@ def __init__(
         truth_probabilities: pd.DataFrame,
         prediction_probabilities: pd.DataFrame,
         truth_weights: pd.DataFrame,
+        validation_metric: Optional[ValidationMetric] = None,
     ) -> None:
         categories = truth_probabilities.columns
 
@@ -61,9 +69,36 @@ def __init__(
         )
 
         self.overall = self.aggregate.at['balanced_accuracy']
-        self.validation = metrics.balanced_multiclass_accuracy(
-            truth_probabilities, prediction_probabilities, truth_weights.validation_weight
-        )
+
+        if validation_metric:
+            if validation_metric == ValidationMetric.BALANCED_ACCURACY:
+                self.validation = metrics.balanced_multiclass_accuracy(
+                    truth_probabilities, prediction_probabilities, truth_weights.validation_weight
+                )
+            elif validation_metric == ValidationMetric.AVERAGE_PRECISION:
+                per_category_ap = pd.Series(
+                    [
+                        metrics.average_precision(
+                            truth_probabilities[category],
+                            prediction_probabilities[category],
+                            truth_weights.validation_weight,
+                        )
+                        for category in categories
+                    ]
+                )
+                self.validation = per_category_ap.mean()
+            elif validation_metric == ValidationMetric.AUC:
+                per_category_auc = pd.Series(
+                    [
+                        metrics.auc(
+                            truth_probabilities[category],
+                            prediction_probabilities[category],
+                            truth_weights.validation_weight,
+                        )
+                        for category in categories
+                    ]
+                )
+                self.validation = per_category_auc.mean()
 
     @staticmethod
     def _category_score(
@@ -153,7 +188,10 @@ def to_dict(self, rocs: bool = True) -> ScoreDict:
 
     @classmethod
     def from_stream(
-        cls, truth_file_stream: TextIO, prediction_file_stream: TextIO
+        cls,
+        truth_file_stream: TextIO,
+        prediction_file_stream: TextIO,
+        validation_metric: Optional[ValidationMetric] = None,
     ) -> ClassificationScore:
         truth_probabilities, truth_weights = parse_truth_csv(truth_file_stream)
         categories = truth_probabilities.columns
@@ -164,16 +202,21 @@ def from_stream(
         sort_rows(truth_probabilities)
         sort_rows(prediction_probabilities)
 
-        score = cls(truth_probabilities, prediction_probabilities, truth_weights)
+        score = cls(truth_probabilities, prediction_probabilities, truth_weights, validation_metric)
         return score
 
     @classmethod
     def from_file(
-        cls, truth_file: pathlib.Path, prediction_file: pathlib.Path
+        cls,
+        truth_file: pathlib.Path,
+        prediction_file: pathlib.Path,
+        validation_metric: Optional[ValidationMetric] = None,
     ) -> ClassificationScore:
         with truth_file.open('r') as truth_file_stream, prediction_file.open(
             'r'
         ) as prediction_file_stream:
             return cls.from_stream(
-                cast(TextIO, truth_file_stream), cast(TextIO, prediction_file_stream)
+                cast(TextIO, truth_file_stream),
+                cast(TextIO, prediction_file_stream),
+                validation_metric,
             )

isic_challenge_scoring/types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 
 class ScoreException(Exception):
@@ -9,13 +9,13 @@ class ScoreException(Exception):
 SeriesDict = Dict[str, float]
 DataFrameDict = Dict[str, SeriesDict]
 RocDict = Dict[str, List[float]]
-ScoreDict = Dict[str, Union[float, SeriesDict, DataFrameDict, Dict[str, RocDict]]]
+ScoreDict = Dict[str, Union[float, Optional[float], SeriesDict, DataFrameDict, Dict[str, RocDict]]]
 
 
 @dataclass
 class Score:
     overall: float
-    validation: float
+    validation: Optional[float]
 
     def to_string(self) -> str:
         output = f'Overall: {self.overall}\n'

tests/data/classification/groundtruth/ISIC2018_Task3_GroundTruth.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cc39a6c579a5bd90639705c6a1ad4a326cb3df0626682be1cc693ec30a20ff6
-size 420666
+oid sha256:ece19dc406aa5293cd0185b6872b00149bea68a2106a396343b4db326642dd28
+size 490801

tests/test_classification.py

@@ -1,7 +1,24 @@
-from isic_challenge_scoring.classification import ClassificationScore
+import pytest
+
+from isic_challenge_scoring.classification import ClassificationScore, ValidationMetric
 
 
 def test_score(classification_truth_file_path, classification_prediction_file_path):
     assert ClassificationScore.from_file(
         classification_truth_file_path, classification_prediction_file_path
     )
+
+
+@pytest.mark.parametrize(
+    'validation_metric',
+    [ValidationMetric.AUC, ValidationMetric.BALANCED_ACCURACY, ValidationMetric.AVERAGE_PRECISION],
+)
+def test_score_validation_metric(
+    classification_truth_file_path, classification_prediction_file_path, validation_metric
+):
+    score = ClassificationScore.from_file(
+        classification_truth_file_path,
+        classification_prediction_file_path,
+        validation_metric=validation_metric,
+    )
+    assert isinstance(score.validation, float)