NuClick + Classification (user experience) (#1092)

* Draft for nuclei classification model

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* sync local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* Sync up local changes

Signed-off-by: Sachidanand Alle <[email protected]>

* fix inference

Signed-off-by: Sachidanand Alle <[email protected]>

* Add QuPath Fixes

Signed-off-by: SACHIDANAND ALLE <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Signed-off-by: Sachidanand Alle <[email protected]>
Signed-off-by: SACHIDANAND ALLE <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: SachidanandAlle

monailabel/config.py

@@ -54,6 +54,7 @@ class Settings(BaseSettings):
     MONAI_LABEL_DICOMWEB_READ_TIMEOUT: float = 5.0
 
     MONAI_LABEL_DATASTORE_AUTO_RELOAD: bool = True
+    MONAI_LABEL_DATASTORE_READ_ONLY: bool = False
     MONAI_LABEL_DATASTORE_FILE_EXT: List[str] = [
         "*.nii.gz",
         "*.nii",

monailabel/datastore/local.py

@@ -103,6 +103,7 @@ def __init__(
         datastore_config: str = "datastore_v2.json",
         extensions=("*.nii.gz", "*.nii"),
         auto_reload=False,
+        read_only=False,
     ):
         """
         Creates a `LocalDataset` object
@@ -142,7 +143,8 @@ def __init__(
         os.makedirs(self._datastore.label_path(DefaultLabelTag.ORIGINAL), exist_ok=True)
 
         # reconcile the loaded datastore file with any existing files in the path
-        self._reconcile_datastore()
+        if not read_only:
+            self._reconcile_datastore()
 
         if auto_reload:
             logger.info("Start observing external modifications on datastore (AUTO RELOAD)")

monailabel/interfaces/app.py

@@ -135,6 +135,7 @@ def init_datastore(self) -> Datastore:
             self.studies,
             extensions=settings.MONAI_LABEL_DATASTORE_FILE_EXT,
             auto_reload=settings.MONAI_LABEL_DATASTORE_AUTO_RELOAD,
+            read_only=settings.MONAI_LABEL_DATASTORE_READ_ONLY,
         )
 
     def init_remote_datastore(self) -> Datastore:

monailabel/tasks/infer/basic_infer.py

@@ -14,6 +14,7 @@
 import os
 import time
 from abc import abstractmethod
+from enum import Enum
 from typing import Any, Callable, Dict, Sequence, Tuple, Union
 
 import torch
@@ -29,6 +30,14 @@
 logger = logging.getLogger(__name__)
 
 
+class CallBackTypes(str, Enum):
+    PRE_TRANSFORMS = "PRE_TRANSFORMS"
+    INFERER = "INFERER"
+    INVERT_TRANSFORMS = "INVERT_TRANSFORMS"
+    POST_TRANSFORMS = "POST_TRANSFORMS"
+    WRITER = "WRITER"
+
+
 class BasicInferTask(InferTask):
     """
     Basic Inference Task Helper
@@ -71,12 +80,11 @@ def __init__(
         :param train_mode: Run in Train mode instead of eval (when network has dropouts)
         :param skip_writer: Skip Writer and return data dictionary
         """
+
+        super().__init__(type, labels, dimension, description, config)
+
         self.path = [] if not path else [path] if isinstance(path, str) else path
         self.network = network
-        self.type = type
-        self.labels = [] if labels is None else [labels] if isinstance(labels, str) else labels
-        self.dimension = dimension
-        self.description = description
         self.model_state_dict = model_state_dict
         self.input_key = input_key
         self.output_label_key = output_label_key
@@ -88,15 +96,18 @@ def __init__(
 
         self._networks: Dict = {}
 
-        self._config: Dict[str, Any] = {
-            "device": device_list(),
-            # "result_extension": None,
-            # "result_dtype": None,
-            # "result_compress": False
-            # "roi_size": self.roi_size,
-            # "sw_batch_size": 1,
-            # "sw_overlap": 0.25,
-        }
+        self._config.update(
+            {
+                "device": device_list(),
+                # "result_extension": None,
+                # "result_dtype": None,
+                # "result_compress": False
+                # "roi_size": self.roi_size,
+                # "sw_batch_size": 1,
+                # "sw_overlap": 0.25,
+            }
+        )
+
         if config:
             self._config.update(config)
 
@@ -232,14 +243,20 @@ def inferer(self, data=None) -> Inferer:
             )
         return SimpleInferer()
 
-    def __call__(self, request) -> Union[Dict, Tuple[str, Dict[str, Any]]]:
+    def __call__(
+        self, request, callbacks: Union[Dict[CallBackTypes, Any], None] = None
+    ) -> Union[Dict, Tuple[str, Dict[str, Any]]]:
         """
         It provides basic implementation to run the following in order
             - Run Pre Transforms
             - Run Inferer
+            - Run Invert Transforms
             - Run Post Transforms
             - Run Writer to save the label mask and result params
 
+        You can provide callbacks which can be useful while writing pipelines to consume intermediate outputs
+        Callback function should consume data and return data (modified/updated) e.g. `def my_cb(data): return data`
+
         Returns: Label (File Path) and Result Params (JSON)
         """
         begin = time.time()
@@ -262,28 +279,47 @@ def __call__(self, request) -> Union[Dict, Tuple[str, Dict[str, Any]]]:
             dump_data(req, logger.level)
             data = req
 
+        # callbacks useful in case of pipeliens to consume intermediate output from each of the following stages
+        # callback function should consume data and returns data (modified/updated)
+        callbacks = callbacks if callbacks else {}
+        callback_run_pre_transforms = callbacks.get(CallBackTypes.PRE_TRANSFORMS)
+        callback_run_inferer = callbacks.get(CallBackTypes.INFERER)
+        callback_run_invert_transforms = callbacks.get(CallBackTypes.INVERT_TRANSFORMS)
+        callback_run_post_transforms = callbacks.get(CallBackTypes.POST_TRANSFORMS)
+        callback_writer = callbacks.get(CallBackTypes.WRITER)
+
         start = time.time()
         pre_transforms = self.pre_transforms(data)
         data = self.run_pre_transforms(data, pre_transforms)
+        if callback_run_pre_transforms:
+            data = callback_run_pre_transforms(data)
         latency_pre = time.time() - start
 
         start = time.time()
         data = self.run_inferer(data, device=device)
+        if callback_run_inferer:
+            data = callback_run_inferer(data)
         latency_inferer = time.time() - start
 
         start = time.time()
         data = self.run_invert_transforms(data, pre_transforms, self.inverse_transforms(data))
+        if callback_run_invert_transforms:
+            data = callback_run_invert_transforms(data)
         latency_invert = time.time() - start
 
         start = time.time()
         data = self.run_post_transforms(data, self.post_transforms(data))
+        if callback_run_post_transforms:
+            data = callback_run_post_transforms(data)
         latency_post = time.time() - start
 
         if self.skip_writer:
             return dict(data)
 
         start = time.time()
         result_file_name, result_json = self.writer(data)
+        if callback_writer:
+            data = callback_writer(data)
         latency_write = time.time() - start
 
         latency_total = time.time() - begin
@@ -467,7 +503,8 @@ def writer(self, data, extension=None, dtype=None) -> Tuple[Any, Any]:
             if isinstance(self.labels, dict):
                 label_names = {v: k for k, v in self.labels.items()}
             else:
-                label_names = {v: k for v, k in enumerate(self.labels)}
+                label_names = {v: k for v, k in enumerate(self.labels)} if isinstance(self.labels, Sequence) else None
+
             cw = ClassificationWriter(label=self.output_label_key, label_names=label_names)
             return cw(data)
 

monailabel/tasks/train/handler.py

@@ -32,7 +32,7 @@ def tensor_to_list(d):
         return r
 
     stats: Dict[str, Any] = dict()
-    stats.update(trainer.get_train_stats())
+    stats.update(trainer.get_stats())
     stats["epoch"] = trainer.state.epoch
     stats["start_ts"] = int(start_ts)
 

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/MonaiLabelClient.java

@@ -98,13 +98,19 @@ public static class Strategy {
 		public String description;
 	}
 
+	public static class Trainer {
+		public String description;
+		public Map<String, Object> config;
+	}
+
 	public static class ResponseInfo {
 		public String name;
 		public String description;
 		public String version;
 		public Labels labels;
 		public Map<String, Model> models;
 		public Map<String, Strategy> strategies;
+		public Map<String, Trainer> trainers;
 	}
 
 	public static class ImageInfo {

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/commands/RunInference.java

@@ -19,10 +19,8 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import javax.xml.parsers.ParserConfigurationException;
@@ -50,6 +48,7 @@
 import qupath.lib.plugins.parameters.ParameterList;
 import qupath.lib.regions.ImagePlane;
 import qupath.lib.regions.RegionRequest;
+import qupath.lib.roi.PointsROI;
 import qupath.lib.roi.ROIs;
 import qupath.lib.roi.interfaces.ROI;
 import qupath.lib.scripting.QP;
@@ -80,38 +79,27 @@ public void run() {
 			}
 
 			ResponseInfo info = MonaiLabelClient.info();
-			List<String> names = new ArrayList<String>();
-			Map<String, String[]> labels = new HashMap<String, String[]>();
-			for (String n : info.models.keySet()) {
-				names.add(n);
-				labels.put(n, info.models.get(n).labels.labels());
-			}
+			List<String> names = Arrays.asList(info.models.keySet().toArray(new String[0]));
 
-			ParameterList list = new ParameterList();
 			if (selectedModel == null || selectedModel.isEmpty()) {
 				selectedModel = names.isEmpty() ? "" : names.get(0);
 			}
 
+			ParameterList list = new ParameterList();
 			list.addChoiceParameter("Model", "Model Name", selectedModel, names);
 			list.addStringParameter("Location", "Location (x,y,w,h)", Arrays.toString(bbox));
 			list.addIntParameter("TileSize", "TileSize", tileSize);
 
-			boolean override = !info.models.get(selectedModel).nuclick;
-			list.addBooleanParameter("Override", "Override", override);
-
 			if (Dialogs.showParameterDialog("MONAILabel", list)) {
 				String model = (String) list.getChoiceParameterValue("Model");
 				bbox = Utils.parseStringArray(list.getStringParameterValue("Location"));
-				override = list.getBooleanParameterValue("Override").booleanValue();
 				tileSize = list.getIntParameterValue("TileSize").intValue();
 
 				selectedModel = model;
 				selectedBBox = bbox;
 				selectedTileSize = tileSize;
 
-				boolean validateClicks = info.models.get(selectedModel).nuclick;
-				runInference(model, new HashSet<String>(Arrays.asList(labels.get(model))), bbox, tileSize, imageData,
-						override, validateClicks);
+				runInference(model, info, bbox, tileSize, imageData);
 			}
 		} catch (Exception ex) {
 			ex.printStackTrace();
@@ -124,13 +112,16 @@ ArrayList<Point2> getClicks(String name, ImageData<BufferedImage> imageData, ROI
 		List<PathObject> objs = imageData.getHierarchy().getFlattenedObjectList(null);
 		ArrayList<Point2> clicks = new ArrayList<Point2>();
 		for (int i = 0; i < objs.size(); i++) {
-			String pname = objs.get(i).getPathClass() == null ? "" : objs.get(i).getPathClass().getName();
-			if (pname.equalsIgnoreCase(name)) {
-				ROI r = objs.get(i).getROI();
-				List<Point2> points = r.getAllPoints();
-				for (Point2 p : points) {
-					if (monaiLabelROI.contains(p.getX(), p.getY())) {
-						clicks.add(new Point2(p.getX() - offsetX, p.getY() - offsetY));
+			var obj = objs.get(i);
+			String pname = obj.getPathClass() == null ? "" : obj.getPathClass().getName();
+			if (name.isEmpty() || pname.equalsIgnoreCase(name)) {
+				ROI r = obj.getROI();
+				if (r instanceof PointsROI) {
+					List<Point2> points = r.getAllPoints();
+					for (Point2 p : points) {
+						if (monaiLabelROI.contains(p.getX(), p.getY())) {
+							clicks.add(new Point2(p.getX() - offsetX, p.getY() - offsetY));
+						}
 					}
 				}
 			}
@@ -140,12 +131,17 @@ ArrayList<Point2> getClicks(String name, ImageData<BufferedImage> imageData, ROI
 		return clicks;
 	}
 
-	private void runInference(String model, Set<String> labels, int[] bbox, int tileSize,
-			ImageData<BufferedImage> imageData, boolean override, boolean validateClicks)
+	private void runInference(String model, ResponseInfo info, int[] bbox, int tileSize,
+			ImageData<BufferedImage> imageData)
 			throws SAXException, IOException, ParserConfigurationException, InterruptedException {
 		logger.info("MONAILabel:: Running Inference...");
-		logger.info("MONAILabel:: Model: " + model + "; override: " + override + "; clicks:" + validateClicks
-				+ "; Labels: " + labels);
+
+		boolean isNuClick = info.models.get(model).nuclick;
+		boolean override = !isNuClick;
+		boolean validateClicks = isNuClick;
+		var labels = new HashSet<String>(Arrays.asList(info.models.get(model).labels.labels()));
+
+		logger.info("MONAILabel:: Model: " + model + "; Labels: " + labels);
 
 		Path imagePatch = null;
 		try {
@@ -178,30 +174,38 @@ private void runInference(String model, Set<String> labels, int[] bbox, int tile
 				req.location[0] = req.location[1] = 0;
 				req.size[0] = req.size[1] = 0;
 
-				var fg = getClicks("Positive", imageData, roi, offsetX, offsetY);
-				var bg = getClicks("Negative", imageData, roi, offsetX, offsetY);
-				if (validateClicks) {
-					if (fg.size() == 0 && bg.size() == 0) {
-						Dialogs.showErrorMessage("MONAILabel",
-								"Need atleast one Postive/Negative annotation/click point within the ROI");
-						return;
-					}
-					if (roi.getBoundsHeight() < 128 || roi.getBoundsWidth() < 128) {
-						Dialogs.showErrorMessage("MONAILabel",
-								"Min Height/Width of ROI should be more than 128");
-						return;
-					}
-				}
-
-				req.params.addClicks(fg, true);
-				req.params.addClicks(bg, false);
 
 				imagePatch = java.nio.file.Files.createTempFile("patch", ".png");
 				imageFile = imagePatch.toString();
 				var requestROI = RegionRequest.createInstance(imageData.getServer().getPath(), 1, roi);
 				ImageWriterTools.writeImageRegion(imageData.getServer(), requestROI, imageFile);
 			}
 
+			ArrayList<Point2> fg = new ArrayList<>();
+			ArrayList<Point2> bg = new ArrayList<>();
+			if (isNuClick) {
+				fg = getClicks("", imageData, roi, offsetX, offsetY);
+			} else {
+				fg = getClicks("Positive", imageData, roi, offsetX, offsetY);
+				bg = getClicks("Negative", imageData, roi, offsetX, offsetY);
+			}
+
+			if (validateClicks) {
+				if (fg.size() == 0 && bg.size() == 0) {
+					Dialogs.showErrorMessage("MONAILabel",
+							"Need atleast one Postive/Negative annotation/click point within the ROI");
+					return;
+				}
+				if (roi.getBoundsHeight() < 128 || roi.getBoundsWidth() < 128) {
+					Dialogs.showErrorMessage("MONAILabel",
+							"Min Height/Width of ROI should be more than 128");
+					return;
+				}
+			}
+			req.params.addClicks(fg, true);
+			req.params.addClicks(bg, false);
+
+
 			Document dom = MonaiLabelClient.infer(model, image, imageFile, sessionId, req);
 			NodeList annotation_list = dom.getElementsByTagName("Annotation");
 			int count = updateAnnotations(labels, annotation_list, roi, imageData, override, offsetX, offsetY);
@@ -227,6 +231,20 @@ private int updateAnnotations(Set<String> labels, NodeList annotation_list, ROI
 					}
 				}
 			}
+		} else {
+			List<PathObject> objs = imageData.getHierarchy().getFlattenedObjectList(null);
+			for (int i = 0; i < objs.size(); i++) {
+				var obj = objs.get(i);
+				ROI r = obj.getROI();
+				if (r instanceof PointsROI) {
+					String pname = obj.getPathClass() == null ? "" : obj.getPathClass().getName();
+					if (pname.equalsIgnoreCase("Positive") || pname.equalsIgnoreCase("Negative")) {
+						continue;
+					}
+					imageData.getHierarchy().removeObjectWithoutUpdate(obj, false);
+				}
+			}
+			QP.fireHierarchyUpdate(imageData.getHierarchy());
 		}
 
 		int count = 0;