rejection of invalid particles already during particle detection

analysis/particleAndMeasurement.py

@@ -32,6 +32,7 @@ class Particle(object):
         self.measurements = []
         self.color = None
         self.shape = None
+        self.wasManuallyEdited = False   #useful tag for extracting data that can now be considered reliable and obviously was classfiied wrong before by the algorithms
     
     def addMeasurement(self, refToMeasurement):
         refToMeasurement.assignedParticle = self

analysis/particleCharacterization.py

@@ -27,7 +27,7 @@ from copy import deepcopy
 from .particleClassification.colorClassification import ColorClassifier
 from .particleClassification.shapeClassification import ShapeClassifier
 from segmentation import closeHolesOfSubImage
-from errors import NotConnectedContoursError, InvalidParticleError
+from errors import InvalidParticleError
 
 class ParticleStats(object):
     longSize = None
@@ -46,16 +46,19 @@ def particleIsValid(particle):
     return True
     
 
-def getParticleStatsWithPixelScale(cnt, pixelscale, fullimage, dataset):
-    newStats = ParticleStats()
+def getParticleStatsWithPixelScale(cnt, fullimage, dataset):
+    pixelscale = dataset.getPixelScale()
     
+    newStats = ParticleStats()
     newStats.longSize, newStats.shortSize, newStats.area = getContourStats(cnt)
     newStats.longSize *= pixelscale
     newStats.shortSize *= pixelscale
     newStats.area *= (pixelscale**2)
-
+    
+    if 0 in [newStats.longSize, newStats.shortSize, newStats.area]:
+        raise InvalidParticleError
+    
     newStats.height = getParticleHeight(cnt, dataset)    
-    print('newHeight =', newStats.height)
     newStats.shape = getParticleShape(cnt, newStats.height)
     
     partImg = getParticleImageFromFullimage(cnt, fullimage)

analysis/particleContainer.py

@@ -98,14 +98,9 @@ class ParticleContainer(object):
         
     def setParticleStats(self, particlestats):
         assert len(self.particles) == len(particlestats)
-        #particlestats is list of [long, short, longellipse, shortellipse, cv2.contourArea(cnt)]
         for index, particle in enumerate(self.particles):
-            particle.longSize_box = float(particlestats[index][0])
-            particle.shortSize_box = float(particlestats[index][1])
-            particle.longSize_ellipse = float(particlestats[index][2])
-            particle.shortSize_ellipse = float(particlestats[index][3])
-            particle.area = float(particlestats[index][4])
-    
+            particle.__dict__.update(particlestats[index].__dict__)
+            
     def testForInconsistentParticles(self): #i.e., particles that have multiple measurements with different assignments
         self.inconsistentParticles = []
         for particle in self.particles:
@@ -146,10 +141,6 @@ class ParticleContainer(object):
             scanIndex = meas.getScanIndex()
             meas.setHQI(hqiList[scanIndex])
     
-    def reassignParticleToAssignment(self, particleIndex, newAssignment):
-        particle = self.getParticleOfIndex(particleIndex)
-        particle.setAllSpectraToNewAssignment(newAssignment)
-    
     def getParticleOfIndex(self, index):
         try:
             particle = self.particles[index]
@@ -248,6 +239,12 @@ class ParticleContainer(object):
             colors.append(particle.color)
         return colors
     
+    def getShapesOfAllParticles(self):
+        shapes = []
+        for particle in self.particles:
+            shapes.append(particle.shape)
+        return shapes
+    
     def getParticleColorByIndex(self, particleIndex):
         particle = self.getParticleOfIndex(particleIndex)
         return particle.color
@@ -289,13 +286,20 @@ class ParticleContainer(object):
         final_typehistogram = {i[0]: i[1] for i in sorted_typehistogram}
         return final_typehistogram
     
+    def reassignParticleToAssignment(self, particleIndex, newAssignment):
+        particle = self.getParticleOfIndex(particleIndex)
+        particle.setAllSpectraToNewAssignment(newAssignment)
+        particle.wasManuallyEdited = True
+    
     def changeParticleColor(self, index, newColor):
         particle = self.getParticleOfIndex(index)
         particle.color = newColor
+        particle.wasManuallyEdited = True
     
     def changeParticleShape(self, index, newShape):
         particle = self.getParticleOfIndex(index)
         particle.shape = newShape
+        particle.wasManuallyEdited = True
     
     def addMergedParticle(self, particleIndices, newContour, newStats, newAssignment=None):
         newParticle = Particle()
@@ -310,7 +314,7 @@ class ParticleContainer(object):
                 newParticle.addMeasurement(meas)
 
         newParticle.__dict__.update(newStats.__dict__)
-
+        newParticle.wasManuallyEdited = True
         self.particles.append(newParticle)
         print('added new particle')
         

analysis/particleEditor.py

@@ -227,8 +227,7 @@ class ParticleEditor(QtCore.QObject):
             self.particlePainter = None
     
     def mergeParticlesInParticleContainerAndSampleView(self, indices, newContour, assignment):
-        pixelscale = self.viewparent.dataset.getPixelScale()
-        stats = pc.getParticleStatsWithPixelScale(newContour, pixelscale, self.viewparent.imgdata, self.viewparent.dataset)
+        stats = pc.getParticleStatsWithPixelScale(newContour, self.viewparent.imgdata, self.viewparent.dataset)
         
         self.viewparent.addParticleContourToIndex(newContour, len(self.viewparent.contourItems)-1)
         self.particleContainer.addMergedParticle(indices, newContour, stats, newAssignment=assignment)
@@ -240,7 +239,6 @@ class ParticleEditor(QtCore.QObject):
         self.viewparent.resetContourIndices()
         self.particleContainer.resetParticleIndices()
         self.particleAssignmentChanged.emit()
-        #TODO: INCLUDE SANITY CHECK!!!!!!!!!
     
     @QtCore.pyqtSlot(list, str)
     def changeParticleColors(self, contourIndices, newColor):

analysis/sqlexport.py

@@ -42,6 +42,7 @@ class SQLExport(QtWidgets.QDialog):
         self.longSizes = np.round(self.particleContainer.getSizesOfAllParticles())
         self.shortSize = np.round(self.particleContainer.getShortSizesOfAllParticles())
         self.colors = self.particleContainer.getColorsOfAllParticles()
+        self.shapes = self.particleContainer.getShapesOfAllParticles()
         self.spectra = self.particleContainer.getSpectraFromDisk()
         
         self.particleImages = None
@@ -219,6 +220,7 @@ class SQLExport(QtWidgets.QDialog):
                         usedCols['Analyst'] = str(self.analystIndices[self.analystSelector.currentIndex()])
                         usedCols['Size_fraction'] = self.getSizeFraction(sizeCategories, self.longSizes[polymInd])
                         usedCols['Colour'] = self.colors[polymInd]
+                        usedCols['Shape'] = self.shapes[polymInd]
                         usedCols[sizeCols[0]] = str(self.longSizes[polymInd])
                         usedCols[sizeCols[1]] = str(self.shortSize[polymInd])
                         if self.particleImages is not None:

dataset.py

@@ -42,15 +42,14 @@ def loadData(fname):
     return retds
 
 def saveData(dataset, fname):
-    pass
-#    with open(fname, "wb") as fp:
-#        # zvalimg is rather large and thus it is saved separately in a tif file 
-#        # only onces after its creation
-#        zvalimg = dataset.zvalimg
-#        if zvalimg is not None:
-#            dataset.zvalimg = "saved"
-#        pickle.dump(dataset, fp, protocol=-1)
-#        dataset.zvalimg = zvalimg
+    with open(fname, "wb") as fp:
+        # zvalimg is rather large and thus it is saved separately in a tif file 
+        # only onces after its creation
+        zvalimg = dataset.zvalimg
+        if zvalimg is not None:
+            dataset.zvalimg = "saved"
+        pickle.dump(dataset, fp, protocol=-1)
+        dataset.zvalimg = zvalimg
 
 def arrayCompare(a1, a2):
     if a1.shape!=a2.shape:

detectionview.py

@@ -569,14 +569,16 @@ class ParticleDetectionView(QtWidgets.QWidget):
         if showname is not None:
             stepImg, imgtype = self.seg.apply2Image(img, self.imglabel.seedpoints,
                                                     self.imglabel.seeddeletepoints,
-                                                    seedradius,
+                                                    seedradius, 
+                                                    self.dataset,
                                                     return_step=showname)
             self.imglabel.showStep(stepImg, imgtype)
         else:
             measurementpoints, contours, particlestats = self.seg.apply2Image(img,
                                                   self.imglabel.seedpoints,
                                                   self.imglabel.seeddeletepoints,
-                                                  seedradius)
+                                                  seedradius,
+                                                  self.dataset)
             self.imglabel.updateDetectionResults(contours, measurementpoints)
             
     @QtCore.pyqtSlot()
@@ -651,32 +653,37 @@ class ParticleDetectionView(QtWidgets.QWidget):
         measurementPoints, contours, particlestats = self.seg.apply2Image(self.img,
                                                                           seedpoints,
                                                                           deletepoints,
-                                                                          seedradius)
+                                                                          seedradius,
+                                                                          self.dataset)
         if measurementPoints is None: # computation was canceled
             return
-        if self.dataset is not None:
-            self.dataset.ramanscandone = False
-            numParticles = len(contours)
-            particleContainer = self.dataset.particleContainer
-            
-            particleContainer.initializeParticles(numParticles)
-            particleContainer.setParticleContours(contours)
-            particleContainer.setParticleStats(particlestats)
-            particleContainer.applyPixelScaleToParticleStats(self.dataset.getPixelScale())
-            
-            for particleIndex in measurementPoints.keys():
-                measPoints = measurementPoints[particleIndex]
-                for index, point in enumerate(measPoints):
-                    curParticle = particleContainer.getParticleOfIndex(particleIndex)
-                    indexOfNewMeas = particleContainer.addEmptyMeasurement()
-                    particleContainer.setMeasurementPixelCoords(indexOfNewMeas, point.x, point.y)
-                    curParticle.addMeasurement(particleContainer.measurements[indexOfNewMeas])
-                    
-            self.dataset.particleDetectionDone = True
-            self.dataset.mode = "prepareraman"
-            self.dataset.save()
         
+        if self.dataset is not None:
+            self.applyResultsToDataset(measurementPoints, contours, particlestats)
         self.threadrunning = False
+    
+    def applyResultsToDataset(self, measurementPoints, contours, particlestats):
+        self.dataset.ramanscandone = False
+        
+        particleContainer = self.dataset.particleContainer
+        
+        numParticles = len(contours)
+        particleContainer.initializeParticles(numParticles)
+        particleContainer.setParticleContours(contours)
+        particleContainer.setParticleStats(particlestats)
+#        particleContainer.applyPixelScaleToParticleStats(self.dataset.getPixelScale())
+        
+        for particleIndex in measurementPoints.keys():
+            measPoints = measurementPoints[particleIndex]
+            for index, point in enumerate(measPoints):
+                curParticle = particleContainer.getParticleOfIndex(particleIndex)
+                indexOfNewMeas = particleContainer.addEmptyMeasurement()
+                particleContainer.setMeasurementPixelCoords(indexOfNewMeas, point.x, point.y)
+                curParticle.addMeasurement(particleContainer.measurements[indexOfNewMeas])
+                
+        self.dataset.particleDetectionDone = True
+        self.dataset.mode = "prepareraman"
+        self.dataset.save()
         
         
 if __name__ == "__main__":

legacyConvert.py

@@ -115,7 +115,16 @@ def transferParticleStatsToParticleContainer(dset):
  
     dset.particleContainer.initializeParticles(len(dset.particlestats))
     dset.particleContainer.setParticleContours(dset.particlecontours)
-    dset.particleContainer.setParticleStats(dset.particlestats)
+#    dset.particleContainer.setParticleStats(dset.particlestats)
+    assert len(dset.particleContainer.particles) == len(dset.particlestats)
+    #particlestats is list of [long, short, longellipse, shortellipse, cv2.contourArea(cnt)]
+    for index, particle in enumerate(dset.particleContainer.particles):
+        particle.longSize_box = float(dset.particlestats[index][0])
+        particle.shortSize_box = float(dset.particlestats[index][1])
+        particle.longSize_ellipse = float(dset.particlestats[index][2])
+        particle.shortSize_ellipse = float(dset.particlestats[index][3])
+        particle.area = float(dset.particlestats[index][4])
+    
     dset.particleContainer.applyPixelScaleToParticleStats(dset.getPixelScale())
     
     dset.particleContainer.clearMeasurements()

ramanscanui.py

@@ -99,9 +99,6 @@ class RamanScanUI(QtWidgets.QWidget):
         
         self.params = []
         for param in self.ramanctrl.ramanParameters:
-#            if param.dtype == 'selectBtn':
-#                self.params.append(QtWidgets.QPushButton(str(param.value)))   
-#                self.params[-1].released.connect(self.makeGetFnameLambda('Select template file', self.ramanctrl.measTemplatePath, param.openFileType, self.params[-1]))
             if param.dtype == 'int':
                 self.params.append(QtWidgets.QSpinBox())
                 self.params[-1].setMinimum(param.minVal)
@@ -231,7 +228,7 @@ class RamanScanUI(QtWidgets.QWidget):
 
         if reply == QtWidgets.QMessageBox.Yes:
             self.dataset.mode = "ramanscan"
-            for measIndex, ramanScanIndex in enumerate(cmin):
+            for ramanScanIndex, measIndex in enumerate(cmin):
                 self.dataset.particleContainer.setMeasurementScanIndex(measIndex, ramanScanIndex)
             
             self.view.saveDataSet()

sampleview.py

@@ -313,7 +313,8 @@ class SampleView(QtWidgets.QGraphicsView):
         
         if self.particlePainter is None:
             if event.button()==QtCore.Qt.LeftButton:
-                self.checkForContourSelection(event)
+                if self.analysiswidget is not None:
+                    self.checkForContourSelection(event)
                 
                 if self.mode in ["OpticalScan", "RamanScan"] and event.modifiers()==QtCore.Qt.ControlModifier:
                     p0 = self.mapToScene(event.pos())
@@ -387,13 +388,11 @@ class SampleView(QtWidgets.QGraphicsView):
             cnt.update()
             if cnt.particleIndex not in self.selectedParticleIndices:
                 self.selectedParticleIndices.append(cnt.particleIndex)
-#                addParticleInfoBox(cnt.particleIndex)
                 
         def removeContourFromSelection(cnt):
             cnt.isSelected = False
             cnt.update()
             self.selectedParticleIndices.remove(cnt.particleIndex)
-#            removeParticleInfoBox(cnt.particleIndex)
         
         def updateParticleInfoBox(index):
             if self.particleInfoBox is not None:
@@ -497,7 +496,7 @@ class SampleView(QtWidgets.QGraphicsView):
                     self.item.setOpacity(1)
 
             else:
-                self.item.setPiparticleInfoBoxxmap(QtGui.QPixmap())
+                self.item.setPixmap(QtGui.QPixmap())
             if self.mode == "OpticalScan":
                 for i, p in zip(self.dataset.fitindices, self.dataset.fitpoints):
                     p = self.dataset.mapToPixel(p, mode=microscope_mode, force=True)
@@ -571,10 +570,11 @@ class SampleView(QtWidgets.QGraphicsView):
         self.clearItems()
         if self.dataset.particleDetectionDone:
             for meas in self.dataset.particleContainer.measurements:
-                number = meas.ramanScanIndex+1
-                item = RamanScanIndicator(self, number, 20, (meas.pixelcoord_x, meas.pixelcoord_y))
-                self.scene().addItem(item)
-                self.ramanscanitems.append(item)
+                if meas.ramanScanIndex is not None:
+                    number = meas.ramanScanIndex+1
+                    item = RamanScanIndicator(self, number, 20, (meas.pixelcoord_x, meas.pixelcoord_y))
+                    self.scene().addItem(item)
+                    self.ramanscanitems.append(item)
     
     def resetParticleContours(self):
         t0 = time.time()

segmentation.py

@@ -28,10 +28,8 @@ from skimage.feature import peak_local_max
 from skimage.morphology import watershed
 from random import random
 
-try:
-    from analysis.particleCharacterization import getContourStats
-except:
-    print('failed importing getContourStats from segmentation.py')
+from errors import InvalidParticleError
+
 
 def closeHolesOfSubImage(subimg):
     #Add padding to TrehsholdImage
@@ -303,16 +301,7 @@ class Segmentation(object):
             x.append(ind%dist.shape[1]-1)
         return y, x
     
-#    def getSubLabelMap(self, labelMap, label):
-#        oneLabel = labelMap==label
-#        i, j = np.arange(labelMap.shape[0]), np.arange(labelMap.shape[1])
-#        i1, i2 = i[np.any(oneLabel, axis=1)][[0,-1]]
-#        j1, j2 = j[np.any(oneLabel, axis=0)][[0,-1]]
-#        sub = labelMap[i1:i2+1, j1:j2+1]
-#        sub = (sub == label)*label
-#        return sub, [i1, i2], [j1, j2]   
-    
-    def apply2Image(self, img, seedpoints, deletepoints, seedradius, return_step=None):
+    def apply2Image(self, img, seedpoints, deletepoints, seedradius, dataset, return_step=None):
         t0 = time()
         # convert to gray image and do histrogram normalization        
         gray = self.convert2Gray(img)
@@ -463,6 +452,7 @@ class Segmentation(object):
         if self.cancelcomputation:
             return None, None, None
         
+        from analysis.particleCharacterization import getParticleStatsWithPixelScale     #TODO: AH, this should be imported at beginning of the file, but there it always chrashes..
         particlestats = []
         measurementPoints = {}
         
@@ -474,7 +464,11 @@ class Segmentation(object):
             label = markers[cnt[0,0,1],cnt[0,0,0]]
             if label==0:
                 continue
-            stats = getContourStats(cnt)
+            try:
+                stats = getParticleStatsWithPixelScale(cnt, img, dataset)
+            except InvalidParticleError: 
+                print('invalid contour in detection, skipping partile. Contour is:', cnt)
+                continue
             particlestats.append(stats)
             x0, x1 = cnt[:,0,0].min(), cnt[:,0,0].max()
             y0, y1 = cnt[:,0,1].min(), cnt[:,0,1].max()
@@ -488,12 +482,8 @@ class Segmentation(object):
                 measurementPoints[particleIndex].append(newMeasPoint)
             particleIndex += 1
                 
-        print(len(np.unique(markers))-1, len(contours))
-        print("stats")
-                
         if return_step is not None:
             raise NotImplementedError(f"this particular return_step: {return_step} is not implemented yet")
-        print("contours")
         
         tf = time()
         print("particle detection took:", tf-t0, "seconds")