Complete cell-SAM with two channel selection

.gitignore

@@ -107,6 +107,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+deepcell-env/
 
 # Spyder project settings
 .spyderproject

backend/.gitignore

@@ -0,0 +1,5 @@
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml

backend/deepcell_label/client.py

@@ -0,0 +1,26 @@
+import asyncio
+import pickle
+import zlib
+
+import websockets
+from flask import current_app
+
+
+async def perform_send(to_send):
+    uri = 'ws://131.215.2.187:8765'
+    async with websockets.connect(uri, ping_interval=None) as websocket:
+        data = {'img': to_send}
+        print(uri)
+        pkt = zlib.compress(pickle.dumps(data))
+        await websocket.send(pkt)
+        print('sent')
+        pkt_received = await websocket.recv()
+        print('received')
+        mask = pickle.loads(zlib.decompress(pkt_received))
+        return mask
+
+
+def send_to_server(to_send):
+    current_app.logger.info('Sent to server to generate mask for cellSAM')
+    mask = asyncio.run(perform_send(to_send))
+    return mask

backend/deepcell_label/label.py

@@ -13,6 +13,8 @@
 from skimage.morphology import dilation, disk, erosion, flood, square
 from skimage.segmentation import morphological_chan_vese, watershed
 
+from deepcell_label.client import send_to_server
+
 
 class Edit(object):
     """
@@ -76,6 +78,9 @@ def load(self, labels_zip):
             self.action = edit['action']
             self.height = edit['height']
             self.width = edit['width']
+            self.d1, self.d2, self.d3, self.d4 = [
+                edit[k] for k in ['d1', 'd2', 'd3', 'd4']
+            ]
             self.args = edit.get('args', None)
             # TODO: specify write mode per cell?
             self.write_mode = edit.get('writeMode', 'overlap')
@@ -102,7 +107,10 @@ def load(self, labels_zip):
         if 'raw.dat' in zf.namelist():
             with zf.open('raw.dat') as f:
                 raw = np.frombuffer(f.read(), np.uint8)
-                self.raw = np.reshape(raw, (self.width, self.height))
+                # self.rawOriginal = np.reshape(raw, (self.d1, self.d2, self.d3, self.d4))[:, 0, :, :]
+                self.rawOriginal = np.reshape(raw, (self.d1, self.d2, self.d3, self.d4))
+                # self.rawOriginal = np.reshape(self.rawOriginal, (self.d3, self.d4, self.d1))
+                self.raw = self.rawOriginal[0][0]
         elif self.action in self.raw_required:
             raise ValueError(
                 f'Include raw array in raw.json to use action {self.action}.'
@@ -418,3 +426,36 @@ def action_dilate(self, cell):
         mask = self.get_mask(cell)
         dilated = dilation(mask, square(3))
         self.add_mask(dilated, cell)
+
+    def action_select_channels(self, channels):
+        self.nuclear_channel = int(channels[0])
+        self.wholecell_channel = int(channels[1])
+
+    def action_segment_all(self, channels):
+        nuclear_channel = channels[0]
+        wholecell_channel = channels[1]
+        to_send = []
+        if self.d1 == 1:
+            to_send = self.raw.reshape(self.d3, self.d4, self.d1)
+        elif self.d1 > 1:
+            nuclear = self.rawOriginal[nuclear_channel][0]
+            wholecell = self.rawOriginal[wholecell_channel][0]
+            to_send = np.stack([nuclear, wholecell], axis=-1)
+        mask = send_to_server(to_send)
+        self.labels = mask.astype(np.int32)
+        if len(self.labels.shape) == 2:
+            self.labels = np.expand_dims(np.expand_dims(self.labels, 0), 3)
+        cells = []
+        for t in range(self.labels.shape[0]):
+            for c in range(self.labels.shape[-1]):
+                for value in np.unique(self.labels[t, :, :, c]):
+                    if value != 0:
+                        cells.append(
+                            {
+                                'cell': int(value),
+                                'value': int(value),
+                                't': int(t),
+                                'c': int(c),
+                            }
+                        )
+        self.cells = cells

backend/deepcell_label/loaders.py

@@ -14,6 +14,7 @@
 
 import magic
 import numpy as np
+from flask import current_app
 from PIL import Image
 from tifffile import TiffFile, TiffWriter
 
@@ -42,6 +43,7 @@ def __init__(self, image_file=None, label_file=None, axes=None):
         self.embeddings = None
 
         self.image_file = image_file
+        current_app.logger.info(image_file)
         self.label_file = label_file if label_file else image_file
         self.axes = axes
 
@@ -87,6 +89,9 @@ def write_images(self):
         """
         X = self.X
         if X is not None:
+            if len(X.shape) == 3:
+                X = np.expand_dims(X, 0)
+                self.X = X
             # Move channel axis
             X = np.moveaxis(X, -1, 1)
             images = io.BytesIO()
@@ -107,6 +112,9 @@ def write_images(self):
     def write_segmentation(self):
         """Writes segmentation to y.ome.tiff in the output zip."""
         y = self.y
+        if len(y.shape) == 2:
+            y = np.expand_dims(np.expand_dims(y, 0), 3)
+            self.y = y
         if y.shape[:-1] != self.X.shape[:-1]:
             raise ValueError(
                 'Segmentation shape %s is incompatible with image shape %s'
@@ -147,18 +155,19 @@ def write_cells(self):
         """Writes cells to cells.json in the output zip."""
         if self.cells is None:
             cells = []
-            for t in range(self.y.shape[0]):
-                for c in range(self.y.shape[-1]):
-                    for value in np.unique(self.y[t, :, :, c]):
-                        if value != 0:
-                            cells.append(
-                                {
-                                    'cell': int(value),
-                                    'value': int(value),
-                                    't': int(t),
-                                    'c': int(c),
-                                }
-                            )
+            # currently disable instant segmentation upon loading
+            # for t in range(self.y.shape[0]):
+            #     for c in range(self.y.shape[-1]):
+            #         for value in np.unique(self.y[t, :, :, c]):
+            #             if value != 0:
+            #                 cells.append(
+            #                     {
+            #                         'cell': int(value),
+            #                         'value': int(value),
+            #                         't': int(t),
+            #                         'c': int(c),
+            #                     }
+            #                 )
             self.cells = cells
         self.zip.writestr('cells.json', json.dumps(self.cells))
 

backend/requirements.txt

@@ -10,9 +10,10 @@ mysqlclient~=2.1.0
 numpy
 python-decouple~=3.1
 python-dotenv~=0.19.2
-python-magic~=0.4.25
+# python-magic~=0.4.25
 requests~=2.29.0
 scikit-image~=0.19.0
 sqlalchemy~=1.3.24
 tifffile
 imagecodecs
+websockets

cypress.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  e2e: {
+    setupNodeEvents(on, config) {
+      // implement node event listeners here
+    },
+  },
+};

cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "[email protected]",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })

cypress/support/e2e.js

@@ -0,0 +1,20 @@
+// ***********************************************************
+// This example support/e2e.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')

frontend/src/Project/DisplayControls/LabeledControls/CellsOpacitySlider.js

@@ -3,7 +3,7 @@ import Slider from '@mui/material/Slider';
 // import Tooltip from '@mui/material/Tooltip';
 import { useSelector } from '@xstate/react';
 import React, { useEffect } from 'react';
-import { useLabeled, useMousetrapRef } from '../../ProjectContext';
+import { useLabeled, useMousetrapRef, useLabelMode } from '../../ProjectContext';
 
 let numMounted = 0;
 
@@ -18,6 +18,9 @@ function CellsOpacitySlider() {
 
   const handleDoubleClick = () => labeled.send({ type: 'SET_CELLS_OPACITY', opacity: 0.3 }); // [0.3, 1] for range slider
 
+  const labelMode = useLabelMode();
+  const isCellTypes = useSelector(labelMode, (state) => state.matches('editCellTypes'));
+
   useEffect(() => {
     const listener = (e) => {
       if (e.key === 'z') {
@@ -47,7 +50,7 @@ function CellsOpacitySlider() {
   return (
     <Tooltip title={<kbd>Z</kbd>} placement='right'>
       <Slider
-        value={opacity}
+        value={isCellTypes ? 0 : opacity}
         min={0}
         max={1}
         // track={false}
@@ -57,6 +60,7 @@ function CellsOpacitySlider() {
         onDoubleClick={handleDoubleClick}
         componentsProps={{ input: { ref: inputRef } }}
         sx={{ py: 0, '& .MuiSlider-thumb': { height: 15, width: 15 } }}
+        disabled={isCellTypes}
       />
     </Tooltip>
   );

frontend/src/Project/EditControls/EditControls.js

@@ -6,6 +6,7 @@ import CellControls from './CellControls';
 import CellTypeControls from './CellTypeControls';
 import TrackingControls from './DivisionsControls';
 import SegmentControls from './SegmentControls';
+import SegmentSamControls from './SegmentSamControls';
 
 function TabPanel(props) {
   const { children, value, index, ...other } = props;
@@ -28,14 +29,16 @@ function EditControls() {
   const value = useSelector(labelMode, (state) => {
     return state.matches('editSegment')
       ? 0
-      : state.matches('editCells')
+      : state.matches('editSegmentSam')
       ? 1
-      : state.matches('editDivisions')
+      : state.matches('editCells')
       ? 2
-      : state.matches('editCellTypes')
+      : state.matches('editDivisions')
       ? 3
-      : state.matches('editSpots')
+      : state.matches('editCellTypes')
       ? 4
+      : state.matches('editSpots')
+      ? 5
       : false;
   });
 
@@ -51,15 +54,18 @@ function EditControls() {
         <SegmentControls />
       </TabPanel>
       <TabPanel value={value} index={1}>
-        <CellControls />
+        <SegmentSamControls />
       </TabPanel>
       <TabPanel value={value} index={2}>
-        <TrackingControls />
+        <CellControls />
       </TabPanel>
       <TabPanel value={value} index={3}>
-        <CellTypeControls />
+        <TrackingControls />
       </TabPanel>
       <TabPanel value={value} index={4}>
+        <CellTypeControls />
+      </TabPanel>
+      <TabPanel value={value} index={5}>
         <SpotsControls />
       </TabPanel>
     </Box>

frontend/src/Project/EditControls/EditTabs.js

@@ -13,14 +13,16 @@ function EditTabs() {
   const value = useSelector(labelMode, (state) => {
     return state.matches('editSegment')
       ? 0
-      : state.matches('editCells')
+      : state.matches('editSegmentSam')
       ? 1
-      : state.matches('editDivisions')
+      : state.matches('editCells')
       ? 2
-      : state.matches('editCellTypes')
+      : state.matches('editDivisions')
       ? 3
-      : state.matches('editSpots')
+      : state.matches('editCellTypes')
       ? 4
+      : state.matches('editSpots')
+      ? 5
       : false;
   });
   const handleChange = (event, newValue) => {
@@ -29,15 +31,18 @@ function EditTabs() {
         labelMode.send('EDIT_SEGMENT');
         break;
       case 1:
-        labelMode.send('EDIT_CELLS');
+        labelMode.send('EDIT_SEGMENT_SAM');
         break;
       case 2:
-        labelMode.send('EDIT_DIVISIONS');
+        labelMode.send('EDIT_CELLS');
         break;
       case 3:
-        labelMode.send('EDIT_CELLTYPES');
+        labelMode.send('EDIT_DIVISIONS');
         break;
       case 4:
+        labelMode.send('EDIT_CELLTYPES');
+        break;
+      case 5:
         labelMode.send('EDIT_SPOTS');
         break;
       default:
@@ -72,6 +77,7 @@ function EditTabs() {
         variant='scrollable'
       >
         <Tab sx={{ p: 0.5, minHeight: 0 }} label='Segment' />
+        <Tab sx={{ p: 0.5, minHeight: 0 }} label='Segment-CellSAM' />
         <Tab sx={{ p: 0.5, minHeight: 0 }} label='Cells' />
         <Tab sx={{ p: 0.5, minHeight: 0 }} label='Divisions' />
         <Tab sx={{ p: 0.5, minHeight: 0 }} label='Cell Types' />