TrueMatch Import reworked

analysis/analysisview.py

@@ -301,8 +301,11 @@ class ParticleAnalysis(QtWidgets.QMainWindow):
             self.refSelector.setDisabled(False)
             
     def importTrueMatchResults(self):
+        if self.importWindow is not None:
+            del self.importWindow
+        
         self.importWindow = LoadTrueMatchResults(self.particleContainer, self)
-        self.importWindow.exec()
+#        self.importWindow.exec()
         
     @QtCore.pyqtSlot()
     def applyHQIThresholdToResults(self):
@@ -622,8 +625,9 @@ class ParticleAnalysis(QtWidgets.QMainWindow):
     
     def closeEvent(self, event):
         for window in [self.importWindow, self.dbWin]:
-            try: window.close()
-            except: pass
+            if window is not None:
+                window.close()
+                
         self.viewparent.imparent.particelAnalysisAct.setChecked(False)
         event.accept()
 

analysis/loadresults.py

@@ -21,10 +21,11 @@ If not, see <https://www.gnu.org/licenses/>.
 """
 
 import numpy as np
-from PyQt5 import QtWidgets
+import os
+from PyQt5 import QtWidgets, QtCore
 
-class LoadTrueMatchResults(QtWidgets.QDialog):
-    def __init__(self, particleContainer, parent):
+class LoadTrueMatchResults(QtWidgets.QWidget):
+    def __init__(self, particleContainer, analysisParent):
         super(LoadTrueMatchResults, self).__init__()
         self.setGeometry(400, 400, 200, 300)
         self.setWindowTitle('Get Truematch Results')
@@ -32,8 +33,8 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         self.setLayout(self.layout)
 
         self.particleContainer = particleContainer
-        self.parent = parent
-        self.parent.setDisabled(True)
+        self.analysisParent = analysisParent
+        self.analysisParent.setDisabled(True)
         self.trueMatchResults = None
         self.polymertypes = None
         self.additives = None
@@ -42,7 +43,7 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         
         self.loadBtn = QtWidgets.QPushButton('LoadTrueMatchResults')
         self.loadBtn.resize(self.loadBtn.sizeHint()) 
-        self.loadBtn.clicked.connect(self.loadFileManually)
+        self.loadBtn.clicked.connect(self.getImportFiles)
         
         self.reviewGroup = QtWidgets.QGroupBox('Review Changes')
         self.reviewGroup.setDisabled(True)
@@ -59,93 +60,122 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         
         self.manualPolymers = {}
         self.manualAdditives = {}
-        self.expWindow = None
-        self.additivePlot = None
         self.editEntryWindow = None
+        self.sortWindow = None
 
         self.numFlagForTakeSpectrum = 1
         self.numFlagForSetUnknown = 2
         self.numFlagForPrompt = 3
+        self.show()
     
     def show3FlagsReview(self):
         self.editEntryWindow = ModifyManualEdits(self, self.manualPolymers, self.manualAdditives)        
         self.editEntryWindow.show()
 
-    def loadFileManually(self):
-        dsetpath = self.parent.dataset.path
-        fnames =QtWidgets.QFileDialog.getOpenFileNames(self, 'Select TrueMatch result file', dsetpath, 'text file (*.txt)')[0]
-        if len(fnames) > 1:
-            QtWidgets.QMessageBox.about(self, 'Info', 'The following order of files was loaded. If incorrect, please call a coder!\n{}'.format('\n'.join([fname for fname in fnames])))
+    def getImportFiles(self):
+        dsetpath = self.analysisParent.dataset.path
+        self.fnames =QtWidgets.QFileDialog.getOpenFileNames(self, 'Select TrueMatch result file', dsetpath, 'text file (*.txt)')[0]
         self.trueMatchResults = []
-        for fileindex, fname in enumerate(fnames):
+        if len(self.fnames) > 1:
+            self.sortWindow = SortImportFiles(self, self.fnames)
+        else:
+            self.runCalculations()
+            
+    def runCalculations(self):
+        allResults = []
+        numSpectra = 0
+        for fileindex, fname in enumerate(self.fnames):
             with open(fname) as file:
-                if fileindex == 0:
-                    for line in file:
-                        self.trueMatchResults.append(line)
-                else:       ##for additional files skip first line (header..)
-                    for lineindex, line in enumerate(file):
-                        if lineindex > 0:
-                            self.trueMatchResults.append(line)
-        self.runCalculations()                            
+                currentResults = []
+                for lineIndex, line in enumerate(file):
+                    if lineIndex == 0:
+                        line = line.strip().split(';')[:-1]         #disregard the last entry of each line, as each line ends with ; <- that produces an empty entry...
+                        assert line[0] == 'Search Spectrum Name', 'Assertion Error in loadTrueMatchResults'
+                        #detect, whether one- or multicomponent-search was done
+                        if line[-1] == 'IsMarked':
+                            numhits = np.int(line[-2].split(' ')[-1])
+                            numcomps = 1
+                        else:
+                            numhits = np.int(line[-1].split(' ')[-1])
+                            numcomps = np.int(line[-1].split(' ')[-3])
+                    else:
+                        currentResults.append(line)
+                        numSpectra += 1
+                        
+                allResults.append(currentResults)
+
+        if numSpectra > 0:
+            print('{} components, {} hits per sample, {} spectra'.format(numcomps, numhits, numSpectra))        
+        
+        self.polymertypes = [None]*numSpectra
+        self.hqis = [None]*numSpectra
+        if numcomps > 1:
+            self.additives = [None]*numSpectra
+            self.addhqis = [None]*numSpectra
+            
+        addedIndices = []
+        numSpectraInPreviousBatches = 0
+        
+        for resBatchIndex, resBatch in enumerate(allResults):
+            print('spectra in batch:', len(resBatch))
+            for result in resBatch:
+                specIndex, polymertype, additive, hqi, addhqi = self.interpretEntry(resBatchIndex, numSpectraInPreviousBatches, result, numhits, numcomps)
+                
+                self.polymertypes[specIndex] = polymertype
+                self.hqis[specIndex] = hqi
+                if numcomps > 1:
+                    self.additives[specIndex] = additive
+                    self.addhqis[specIndex] = addhqi
+                    
+                addedIndices.append(specIndex)               
+        
+            numSpectraInPreviousBatches = len(addedIndices)
+            print(numSpectraInPreviousBatches)
     
-    def formatResults(self, rawResults):   #get rid of header line, first data interpretation
-        results = []
-        for index, line in enumerate(rawResults):
-            if index == 0:
-                line = line.strip().split(';')[:-1]         #disregard the last entry of each line, as each line ends with ; <- that produces an empty entry...
-                assert line[0] == 'Search Spectrum Name', 'Assertion Error in loadTrueMatchResults'
-                #detect, whether one- or multicomponent-search was done
-                if line[-1] == 'IsMarked':
-                    numhits = np.int(line[-2].split(' ')[-1])
-                    numcomps = 1
-                else:
-                    numhits = np.int(line[-1].split(' ')[-1])
-                    numcomps = np.int(line[-1].split(' ')[-3])
-            else:
-                results.append(line)
+        assert sorted(addedIndices) == list(range(numSpectra)), f'mismatch in sorted indices in loadResults\bAdded {len(addedIndices)} assignments to {numSpectra} spectra'
+        assert not None in self.polymertypes, 'wrong assignments in loadResults'
+        assert not None in self.hqis, 'wrong assignments in loadResults'
+
+        self.particleContainer.applyAssignmentListToParticleMeasurements(self.polymertypes)
+        self.particleContainer.applyHQIListToParticleMeasurements(self.hqis)
         
-        numspectra = len(results)
-        if numspectra > 0:
-            print('{} components, {} hits per sample, {} spectra'.format(numcomps, numhits, numspectra))
+        if len(self.manualPolymers) > 0:
+            self.reviewGroup.setDisabled(False)
         
-        return results, numspectra, numcomps, numhits
+        self.analysisParent.applyHQIThresholdToResults()
+        self.loadBtn.setText('Data loaded')
+        QtWidgets.QMessageBox.about(self, 'Success', 'Results were imported successfully.\nYou can close this window or modify manual edits.')
     
-    def interpretEntry(self, entry, numhits, numcomps):
-        entry = entry.split(';')
+    def interpretEntry(self, resBatchIndex, numSpectraInPreviousBatches, result, numhits, numcomps):
+        result = result.split(';')
         specIndex, polymertype, additive, hqi, addhqi = None, None, None, None, 0   #assign defaults
         
         #specNameFormat is: SampleName_000_Spec.Data 1 (SpecIndex)
-        specName = entry[0]
-        indexOpenBracket = specName.find('(')
-        indexCloseBracket = specName.find(')')
-        inbrackets = specName[indexOpenBracket+1:indexCloseBracket]
-        try:
-            specIndex = int(inbrackets)
-        except:
-            print(inbrackets)
-            raise
+        specName = result[0]
+        curSpecIndex = self.getSpecIndexFromName(specName)
+        specIndex = numSpectraInPreviousBatches + curSpecIndex
         
         #find yes-flags
-        flags = np.where(np.array(entry) == 'yes')[0]
+        flags = np.where(np.array(result) == 'yes')[0]
         if len(flags) == 0:
             #take highest HQI entry
             if numcomps == 1:
-                polymertype = entry[2]
-                hqi = entry[1]
+                polymertype = result[2]
+                hqi = result[1]
             else:
-                polymertype = entry[5]
-                hqi = entry[1]
-                additive = entry[7]
-                addhqi = entry[6]
+                polymertype = result[5]
+                hqi = result[1]
+                additive = result[7]
+                addhqi = result[6]
 
         elif len(flags) == 1:
             #exactly one flag was placed, take this entry
             if numcomps == 1:
-                polymertype = entry[int(flags[0])-1]
+                polymertype = result[int(flags[0])-1]
                 hqi = 100
             else:
-                polymertype = entry[int(flags[0])+2]
-                additive = entry[int(flags[0])+4]
+                polymertype = result[int(flags[0])+2]
+                additive = result[int(flags[0])+4]
                 hqi = 100
                 addhqi = 100
             
@@ -158,71 +188,94 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
                 
         elif len(flags) == 3:
             hqi = 100
-            if specIndex not in self.manualPolymers:
-                polymertype, ok = QtWidgets.QInputDialog.getText(self, 'Name of main component', 'Spectrum at index {} is:'.format(specIndex))
-                self.manualPolymers[specIndex] = polymertype
+            if (curSpecIndex, resBatchIndex) not in self.manualPolymers:
+                polymertype, ok = QtWidgets.QInputDialog.getText(self, 'Name of main component', 'Spectrum at index {} of result file {} is:'.format(curSpecIndex, resBatchIndex+1))
+                self.manualPolymers[(curSpecIndex, resBatchIndex)] = polymertype
             else:
-                polymertype = self.manualPolymers[specIndex]
+                polymertype = self.manualPolymers[(curSpecIndex, resBatchIndex)]
             if numcomps > 1:
                 addhqi = 100
-                if entry[0] not in self.manualAdditives:
-                    additive, ok = QtWidgets.QInputDialog.getText(self, 'Name of additive', 'Additive at index {} is:'.format(specIndex))
-                    self.manualAdditives[specIndex] = additive
+                if (curSpecIndex, resBatchIndex) not in self.manualAdditives:
+                    additive, ok = QtWidgets.QInputDialog.getText(self, 'Name of additive', 'Additive at index {} of result file {} is:'.format(curSpecIndex, resBatchIndex+1))
+                    self.manualAdditives[(curSpecIndex, resBatchIndex)] = additive
                 else:
-                    additive = self.manualAdditives[specIndex]
+                    additive = self.manualAdditives[(curSpecIndex, resBatchIndex)]
         else: 
             QtWidgets.QMessageBox.about(self, 'Error!', 'No rule for {} flags, found at spectrum index {}'.format(len(flags), specIndex))
             
         return specIndex, polymertype, additive, float(hqi), float(addhqi)
-
-    def runCalculations(self):     
-        self.resultList, numspectra, numcomps, numhits = self.formatResults(self.trueMatchResults)
-        self.polymertypes = [None]*numspectra
-        self.hqis = [None]*numspectra        
-        
-        if numcomps == 1:               #####SINGLE COMPONENT SEARCH
-            self.additives = None
-            self.addhqis = None
-            for index, entry in enumerate(self.resultList):
-                if len(entry) == 0:
-                    del self.resultList[index]
-                else:
-                    specIndex, polymertype, additive, hqi, addhqi = self.interpretEntry(entry, numhits, numcomps)
-                    self.polymertypes[specIndex] = polymertype
-                    self.hqis[specIndex] = hqi
-
-        else:                   #####MULTI-COMPONENT SEARCH
-            self.additives = [None]*numspectra
-            self.addhqis = [None]*numspectra
-
-            for index, entry in enumerate(self.resultList):
-                if len(entry) > 0:
-                    specIndex, polymertype, additive, hqi = self.interpretEntry(entry, numhits, numcomps)
-                    self.polymertypes[specIndex] = polymertype
-                    self.hqis[specIndex] = hqi
-                    self.additives[specIndex] = additive
-                    self.addhqis[specIndex] = addhqi
     
-        assert len(self.polymertypes) == len(self.resultList), 'incorrect number of polymer types added...'
-        assert not None in self.polymertypes, 'wrong assignments in loadResults'
-        assert not None in self.hqis, 'wrong assignments in loadResults'
-
-        self.particleContainer.applyAssignmentListToParticleMeasurements(self.polymertypes)
-        self.particleContainer.applyHQIListToParticleMeasurements(self.hqis)
-        
-        if len(self.manualPolymers) > 0:
-            self.reviewGroup.setDisabled(False)
-        
-        self.parent.applyHQIThresholdToResults()
-        self.loadBtn.setText('Data loaded')
-        QtWidgets.QMessageBox.about(self, 'Success', 'Results were imported successfully.\nYou can close this window or modify manual edits.')
-
+    def getSpecIndexFromName(self, specName):
+        number = ''
+        for i in reversed(range(len(specName))):
+            if specName[i] == ')':
+                continue
+            if specName[i] != '(':
+                number = specName[i] + number
+            else:
+                break
+        return int(number)
+    
     def closeEvent(self, event):
-        #TODO: SANITY CHECK FOR DATA CONSISTENCY!!
-        self.parent.setEnabled(True)
+        for win in [self.editEntryWindow, self.sortWindow]:
+            if win is not None:
+                win.close()
+                
+        self.analysisParent.setEnabled(True)
         event.accept()
 
 
+class SortImportFiles(QtWidgets.QWidget):
+    def __init__(self, loadResultsParent, importFiles):
+        super(SortImportFiles, self).__init__()
+        self.setWindowTitle('Sort Import Files')
+        self.loadResultsParent = loadResultsParent
+        self.loadResultsParent.setDisabled(True)
+        self.importFiles = importFiles
+        self.spinBoxes = []
+        layout = QtWidgets.QGridLayout()
+        self.setLayout(layout)
+        
+        label1 = QtWidgets.QLabel('File name')
+        label1.setMinimumWidth(150)
+        label2 = QtWidgets.QLabel('Order')
+        label2.setMinimumWidth(50)
+        
+        layout.addWidget(label1, 0, 0, QtCore.Qt.AlignLeft)
+        layout.addWidget(label2, 0, 1, QtCore.Qt.AlignRight)
+        
+        for fileIndex, importFile in enumerate(self.importFiles):
+            layout.addWidget(QtWidgets.QLabel(os.path.basename(importFile)), 1+fileIndex, 0, QtCore.Qt.AlignLeft)
+            newSpinBox = QtWidgets.QSpinBox()
+            newSpinBox.setMaximumWidth(50)
+            newSpinBox.setMinimum(1)
+            newSpinBox.setMaximum(len(self.importFiles))
+            newSpinBox.setValue(fileIndex+1)
+            layout.addWidget(newSpinBox, 1+fileIndex, 1, QtCore.Qt.AlignRight)
+            self.spinBoxes.append(newSpinBox)
+        
+        self.acceptBtn = QtWidgets.QPushButton('Accept and Close')
+        self.acceptBtn.clicked.connect(self.sortResultFiles)
+        layout.addWidget(self.acceptBtn, fileIndex+2, 0, QtCore.Qt.AlignLeft)
+        
+        self.show()
+    
+    def sortResultFiles(self):
+        order = [box.value()-1 for box in self.spinBoxes]
+        if list(np.unique(order)) != list(range(len(self.spinBoxes))):
+            QtWidgets.QMessageBox.critical(self, 'Error', 'Invalid order indicated')
+            return
+        else:
+            newFiles = []
+            for index in order:
+                newFiles.append(self.importFiles[index])
+            
+            self.loadResultsParent.fnames = newFiles
+            self.loadResultsParent.runCalculations()
+            self.loadResultsParent.setDisabled(False)
+            self.close()
+            
+            
 class ModifyManualEdits(QtWidgets.QWidget):
     def __init__(self, parentWindow, polymerEdits, additiveEdits = None):
         super(ModifyManualEdits, self).__init__()
@@ -242,14 +295,13 @@ class ModifyManualEdits(QtWidgets.QWidget):
         groupLayout = QtWidgets.QGridLayout()
 
         #create Labels and LineEdits        
-        for index in self.polymerEdits:
-            self.labels.append(QtWidgets.QLabel('Spectrum index:' + str(index)))
-            print(self.labels[-1].text())
+        for curSpecIndex, resBatchIndex in self.polymerEdits:
+            self.labels.append(QtWidgets.QLabel(f'Index of Spectrum: {curSpecIndex}, in file number: {resBatchIndex+1}'))
             self.polymerLineEdits.append(QtWidgets.QLineEdit())
-            self.polymerLineEdits[-1].setText(self.polymerEdits[index])
+            self.polymerLineEdits[-1].setText(self.polymerEdits[(curSpecIndex, resBatchIndex)])
             if len(self.additiveEdits) > 0:
                self.additiveLineEdits.append(QtWidgets.QLineEdit()) 
-               self.additiveLineEdits[-1].setText(self.additiveEdits[index])
+               self.additiveLineEdits[-1].setText(self.additiveEdits[(curSpecIndex, resBatchIndex)])
         
         for i in range(len(self.labels)):
             groupLayout.addWidget(self.labels[i], i, 0)
@@ -265,10 +317,10 @@ class ModifyManualEdits(QtWidgets.QWidget):
         layout.addWidget(scrollarea)
         
         saveAndCloseBtn = QtWidgets.QPushButton('Save and Exit')
-        saveAndCloseBtn.clicked.connect(self.closeEvent)
+        saveAndCloseBtn.clicked.connect(self.saveAndClose)
         layout.addWidget(saveAndCloseBtn)
 
-    def closeEvent(self, event):
+    def saveAndClose(self):
         #read out LineEdits:
         for index, key in enumerate(self.polymerEdits):
             self.polymerEdits[key] = self.polymerLineEdits[index].text()
@@ -279,4 +331,3 @@ class ModifyManualEdits(QtWidgets.QWidget):
         self.parent.manualPolymers, self.parent.manualAdditives = self.polymerEdits, self.additiveEdits
         self.parent.runCalculations()
         self.close()
-    

analysis/particleCharacterization.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+GEPARD - Gepard-Enabled PARticle Detection
+Copyright (C) 2018  Lars Bittrich and Josef Brandt, Leibniz-Institut für 
+Polymerforschung Dresden e. V. <bittrich-lars@ipfdd.de>    
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program, see COPYING.  
+If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import numpy as np
+import cv2
+
+def getContourStatsWithPixelScale(cnt, pixelscale):
+        long, short, longellipse, shortellipse, area = getContourStats(cnt)
+        return long*pixelscale, short*pixelscale, longellipse*pixelscale, shortellipse*pixelscale, area*pixelscale**2
+
+def getContourStats(cnt):
+    ##characterize particle
+        longellipse, shortellipse = np.nan, np.nan
+        
+        if cnt.shape[0] >= 5:       ##at least 5 points required for ellipse fitting...
+            ellipse = cv2.fitEllipse(cnt)
+            shortellipse, longellipse = ellipse[1]
+
+        rect = cv2.minAreaRect(cnt)
+        long, short = rect[1]
+        if short>long:
+            long, short = short, long
+        
+        area = cv2.contourArea(cnt)
+    
+        return long, short, longellipse, shortellipse, area
\ No newline at end of file

analysis/particleEditor.py

@@ -31,25 +31,8 @@ from PyQt5 import QtWidgets, QtCore
 from copy import deepcopy
 
 from .particlePainter import ParticlePainter
+from .particleCharacterization import getContourStatsWithPixelScale
 
-def getContourStatsWithPixelScale(contours, pixelscale):
-        ##characterize particle
-        longellipse, shortellipse = np.nan, np.nan
-        
-        cnt = contours
-        
-        if cnt.shape[0] >= 5:       ##at least 5 points required for ellipse fitting...
-            ellipse = cv2.fitEllipse(cnt)
-            shortellipse, longellipse = ellipse[1]
-
-        rect = cv2.minAreaRect(cnt)
-        long, short = rect[1]
-        if short>long:
-            long, short = short, long
-        
-        area = cv2.contourArea(cnt)
-    
-        return long*pixelscale, short*pixelscale, longellipse*pixelscale, shortellipse*pixelscale, area*pixelscale**2
 
 
 class ParticleContextMenu(QtWidgets.QMenu):

segmentation.py

@@ -28,6 +28,8 @@ from skimage.feature import peak_local_max
 from skimage.morphology import watershed
 from random import random
 
+from .analysis.particleCharacterization import getContourStats
+
 class Parameter(object):
     def __init__(self, name, dtype, value=None, minval=None, maxval=None, 
                  decimals=0, stepsize=1, helptext=None, show=False, linkedParameter=None):
@@ -282,21 +284,6 @@ class Segmentation(object):
             
         sure_fg = cv2.dilate(sure_fg, np.ones((3, 3)))
         return sure_fg
-
-    def characterizeParticle(self, contours):
-        longellipse, shortellipse = np.nan, np.nan
-        
-        cnt = contours
-        
-        if cnt.shape[0] >= 5:       ##at least 5 points required for ellipse fitting...
-            ellipse = cv2.fitEllipse(cnt)
-            shortellipse, longellipse = ellipse[1]
-        rect = cv2.minAreaRect(cnt)
-        long, short = rect[1]
-        if short>long:
-            long, short = short, long
-    
-        return long, short, longellipse, shortellipse, cv2.contourArea(cnt)
     
     def getMeasurementPoints(self, binParticle, numPoints=1):
         binParticle = cv2.copyMakeBorder(binParticle, 1, 1, 1, 1, 0)
@@ -449,7 +436,6 @@ class Segmentation(object):
             img[np.nonzero(sure_bg)] |= 2
             return img, 1        
         
-        
         dist_transform = cv2.distanceTransform(thresh, cv2.DIST_L2,5)
         print("distanceTransform")
         if self.cancelcomputation:
@@ -464,7 +450,7 @@ class Segmentation(object):
             return None, None, None
         if return_step=="watershed":
             return np.uint8(255*(markers!=0)), 0  
-
+        
         if cv2.__version__ > '3.5':        
             contours, hierarchy = cv2.findContours(markers, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)
         else:
@@ -484,7 +470,8 @@ class Segmentation(object):
             label = markers[cnt[0,0,1],cnt[0,0,0]]
             if label==0:
                 continue
-            particlestats.append(self.characterizeParticle(cnt))
+            stats = getContourStats(cnt)
+            particlestats.append(stats)
             x0, x1 = cnt[:,0,0].min(), cnt[:,0,0].max()
             y0, y1 = cnt[:,0,1].min(), cnt[:,0,1].max()
             subimg = (markers[y0:y1+1,x0:x1+1]).copy()