Excel and SQL Export working (hotfix)

analysis/analysisview.py

@@ -160,11 +160,13 @@ class ParticleAnalysis(QtWidgets.QMainWindow):
         optionsLayout = QtWidgets.QFormLayout()
         
         self.hqiSpinBox = QtWidgets.QDoubleSpinBox()
-        self.hqiSpinBox.setValue(self.dataset.resultParams['minHQI'])
         self.hqiSpinBox.setDecimals(1)
         self.hqiSpinBox.setMinimum(0)
         self.hqiSpinBox.setMaximum(100)
         self.hqiSpinBox.setMaximumWidth(100)
+        minHQI = self.dataset.resultParams['minHQI']
+        if minHQI is not None:
+            self.hqiSpinBox.setValue(minHQI)
         self.hqiSpinBox.valueChanged.connect(self.applyHQIThresholdToResults)
         optionsLayout.addRow(QtWidgets.QLabel('minimum HQI:'), self.hqiSpinBox)
         
@@ -198,9 +200,7 @@ class ParticleAnalysis(QtWidgets.QMainWindow):
         self.layout.addLayout(self.menuLayout)
         self.layout.addLayout(viewLayout)
         
-        minHQI = self.dataset.resultParams['minHQI']
-        if minHQI is not None:
-            self.hqiSpinBox.setValue(minHQI)
+
          
         self.createActions()
         self.createMenus()
@@ -611,7 +611,13 @@ class ParticleAnalysis(QtWidgets.QMainWindow):
         expWin.exec()
             
     def exportToSQL(self):
-        sqlexp = SQLExport(self.particleContainer, self)
+        if len(self.dataset.resultsUploadedToSQL) > 0:
+           reply = QtWidgets.QMessageBox.question(self, 'Warning!',
+                                "The following results were already uploaded:\n\n{}\n\nContinue?".format('\n'.join(self.dataset.resultsUploadedToSQL)),
+                                QtWidgets.QMessageBox.Yes | 
+                                QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
+           if reply == QtWidgets.QMessageBox.Yes:
+                sqlexp = SQLExport(self.dataset)
                 sqlexp.exec()
     
     def closeEvent(self, event):

analysis/excelexport.py

@@ -45,7 +45,6 @@ class ExpExcelDialog(QtWidgets.QDialog):
         self.exportOptions = ['Polymer Type (mandatory)', 'Long Size (µm)', 'Short Size (µm)', 'Area (µm²)', 'HQI', 'Size Classes']
         self.checkBoxes = []
         self.sizeClasses = [5, 10, 20, 50, 100, 1e6]
-        self.directory = self.dataset.path
         
         for index, option in enumerate(self.exportOptions):
             self.checkBoxes.append(QtWidgets.QCheckBox(self))
@@ -76,8 +75,8 @@ class ExpExcelDialog(QtWidgets.QDialog):
     def toExcel(self):
         requiredcolumns = []       
         uniquePolymers = self.particleContainer.getUniquePolymers()
-        polymers = self.particleContainer.getListOfParticleAssignments()
-        sizes = self.particleContainer.getSizesOfAllParticles()
+        polymers = np.array(self.particleContainer.getListOfParticleAssignments())
+        sizes = np.array(self.particleContainer.getSizesOfAllParticles())
         
 
         for box in self.checkBoxes:
@@ -85,13 +84,13 @@ class ExpExcelDialog(QtWidgets.QDialog):
                 if box.text() != 'Size Classes':
                     requiredcolumns.append(box.text())
                     if box.text() == 'Long Size (µm)':
-                        longSizes = sizes
+                        longSizes = np.round(sizes)
                     elif box.text() == 'Short Size (µm)':
-                        shortSizes = self.particleContainer.getShortSizesOfAllParticles()
+                        shortSizes = np.round(np.array(self.particleContainer.getShortSizesOfAllParticles()))
                     elif box.text() == 'HQI':
-                        hqis = self.particleContainer.getListOfHighestHQIs()
+                        hqis = np.array(self.particleContainer.getListOfHighestHQIs())
                     elif box.text() == 'Area (µm²)':
-                        areas = self.particleContainer.getAreasOfAllParticles()
+                        areas = np.array(self.particleContainer.getAreasOfAllParticles())
                     
                 else:
                     requiredcolumns.append('0 - 5 µm')
@@ -101,13 +100,12 @@ class ExpExcelDialog(QtWidgets.QDialog):
                     requiredcolumns.append('50 - 100 µm')
                     requiredcolumns.append('> 100 µm')
         
-        finalData = np.zeros((polymers.shape[0],len(requiredcolumns)-1))
-        polymertypes = [""]*polymers.shape[0]
+        finalData = np.zeros((len(polymers),len(requiredcolumns)-1))
+        polymertypes = [""]*len(polymers)
         rowindex = 0
         for polymer in uniquePolymers:
-            indices = self.particleContainer.getgetIndicesOfParticleType(polymer)
+            indices = polymers == polymer
             numentries = int(np.sum(indices))
-            print("Num:", numentries)
             sys.stdout.flush()
             
             for colindex, column in enumerate(requiredcolumns):
@@ -130,7 +128,7 @@ class ExpExcelDialog(QtWidgets.QDialog):
                         upLimit = self.sizeClasses[classindex]
                         if classindex == 0: lowLimit = 0
                         else: lowLimit = self.sizeClasses[classindex-1]
-                        curSize = self.sizes[dataindex]
+                        curSize = sizes[dataindex]
                         
                         if  curSize > lowLimit and curSize <= upLimit:
                             finalData[rowindex+tableindex, numPrevCols + classindex] = np.int(1)
@@ -140,15 +138,11 @@ class ExpExcelDialog(QtWidgets.QDialog):
             rowindex = rowindex + numentries
             
         #dump into excel file
-        xlsname = self.directory + '//' + self.xlsFileName.text() + '.xlsx'
-        print('exporting excel to:\n file name:  {} in directory: {}'.format(self.xlsFileName.text(), self.directory))
-        validFileName = False
+        xlsname = os.path.join(self.dataset.path, f'{self.xlsFileName.text()}.xlsx')
+        print('exporting excel to:\n file name:  {} in directory: {}'.format(self.xlsFileName.text(), self.dataset.path))
         incr = 1
-        while not validFileName:
-            if not os.path.exists(xlsname):
-                validFileName = True
-            else:
-                xlsname = self.directory + '//' + self.xlsFileName.text() + ' {}.xlsx'.format(incr)
+        while os.path.exists(xlsname):
+            xlsname = os.path.join(self.dataset.path, f'{self.xlsFileName.text()} {incr}.xlsx')
             incr += 1
     
         writer = pd.ExcelWriter(xlsname, engine = 'xlsxwriter')
@@ -159,20 +153,20 @@ class ExpExcelDialog(QtWidgets.QDialog):
         if  '> 100 µm' in requiredcolumns:
             #generate particle statistics report
             header = ['0 - 5 µm', '5 - 10 µm', '10 - 20 µm', '20 - 50 µm', '50 - 100 µm', '> 100 µm']
-            index = np.unique(self.polymers)
             particleclasses = []
             
-            for polymer in index:
-                indices = np.where(self.polymers == polymer)[0]
-                sortind = np.searchsorted([5,10,20,50,100], self.sizes[indices], 'right')
+            for polymer in uniquePolymers:
+                indices = self.particleContainer.getIndicesOfParticleType(polymer)
+                sortind = np.searchsorted([5,10,20,50,100], sizes[indices], 'right')
                 classes = np.bincount(sortind, minlength=6)
                 particleclasses.append(classes)
             
             particleclasses = np.array(particleclasses)
             report = pd.DataFrame(np.array(particleclasses), columns=header,
                                   dtype=int)
-            report.insert(0, 'Polymer Type', index)
+            report.insert(0, 'Polymer Type', uniquePolymers)
             report.insert(len(report.columns), 'Sum total', particleclasses.sum(axis=1))
             report.to_excel(writer, sheet_name = 'Particle Statistics', index=False)
         writer.save()
         self.accept()
+        QtWidgets.QMessageBox.about(self, 'Particles succesfully exported', 'List saved to\n' + str(xlsname))
\ No newline at end of file

analysis/loadresults.py

@@ -44,31 +44,6 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         self.loadBtn.resize(self.loadBtn.sizeHint()) 
         self.loadBtn.clicked.connect(self.loadFileManually)
         
-#        optionsLayout = QtWidgets.QFormLayout()
-#        self.optionsGroup = QtWidgets.QGroupBox('Compute Options')
-#        self.optionsGroup.setDisabled(True)
-#        
-#        self.label1 = QtWidgets.QLabel('HQI-Threshold:')
-#        self.spinbox1 = QtWidgets.QDoubleSpinBox(self)
-#        self.spinbox1.valueChanged.connect(self.runCalculations)
-#        optionsLayout.addRow(self.label1, self.spinbox1)
-        
-#        self.label2 = QtWidgets.QLabel('ComponentThreshold:')
-#        self.spinbox2 = QtWidgets.QDoubleSpinBox(self)
-#        self.spinbox2.valueChanged.connect(self.updateParentSpinboxes)
-#        optionsLayout.addRow(self.label2, self.spinbox2)
-        
-#        for box in [self.spinbox1, self.spinbox2]:
-#        for box in [self.spinbox1]:
-#            box.setValue(50)
-#            box.setDecimals(1)
-#            box.setSingleStep(5)
-#            box.setMinimum(0)
-#            box.setMaximum(100)
-##        self.spinbox2.setValue(30)          #more reasonable...
-#        
-#        self.optionsGroup.setLayout(optionsLayout)
-        
         self.reviewGroup = QtWidgets.QGroupBox('Review Changes')
         self.reviewGroup.setDisabled(True)
         reviewLayout = QtWidgets.QVBoxLayout()
@@ -80,7 +55,6 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         self.reviewGroup.setLayout(reviewLayout)
 
         self.layout.addWidget(self.loadBtn, 0, 0)
-#        self.layout.addWidget(self.optionsGroup, 1, 0)
         self.layout.addWidget(self.reviewGroup, 2, 0)
         
         self.manualPolymers = {}
@@ -144,7 +118,12 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         specName = entry[0]
         indexOpenBracket = specName.find('(')
         indexCloseBracket = specName.find(')')
-        specIndex = int(specName[indexOpenBracket+1:indexCloseBracket])
+        inbrackets = specName[indexOpenBracket+1:indexCloseBracket]
+        try:
+            specIndex = int(inbrackets)
+        except:
+            print(inbrackets)
+            raise
         
         #find yes-flags
         flags = np.where(np.array(entry) == 'yes')[0]
@@ -204,7 +183,6 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
         if numcomps == 1:               #####SINGLE COMPONENT SEARCH
             self.additives = None
             self.addhqis = None
-#            self.spinbox2.setEnabled(False)
             for index, entry in enumerate(self.resultList):
                 if len(entry) == 0:
                     del self.resultList[index]
@@ -241,11 +219,6 @@ class LoadTrueMatchResults(QtWidgets.QDialog):
 
     def closeEvent(self, event):
         #TODO: SANITY CHECK FOR DATA CONSISTENCY!!
-        
-#        if not self.parent.datastats.saveAnalysisResults(minHQI, compHQI):
-#            QtWidgets.QMessageBox.warning(self.parent, 'Error!',
-#                                'Data inconsistency after saving!',
-#                                QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.Ok)
         self.parent.setEnabled(True)
         event.accept()
 

analysis/particleAndMeasurement.py

@@ -83,17 +83,19 @@ class Particle(object):
         else:
             print(f'Error, particle size requested, but not yet set.\nParticle Index is {self.index}')
             raise ValueError
-        assert size is not None, f'Error, size or particle {self.index} is None'
-        return round(size)
+        assert size is not None, f'Error, size of particle {self.index} is None'
+        return size
     
     def getShortParticleSize(self):
         if not np.isnan(self.shortSize_ellipse):
-            return self.shortSize_ellipse
+            size = self.shortSize_ellipse
         elif not np.isnan(self.shortSize_box):
-            return self.shortSize_box
+            size = self.shortSize_box
         else:
             print(f'Error, particle size requested, but not yet set.\nParticle Index is {self.index}')
             raise ValueError
+        assert size is not None, f'Error, short size of particle {self.index} is None'
+        return size
     
     def getNumberOfMeasurements(self):
         return len(self.measurements)

analysis/particleContainer.py

@@ -33,7 +33,6 @@ class ParticleContainer(object):
         self.datasetParent = datasetParent
         self.particles = []
         self.measurements = []
-        self.spectra = None
         self.inconsistentParticles = []
         
         self.typeHistogram = None
@@ -83,7 +82,6 @@ class ParticleContainer(object):
                 
             if spectra is not None:
                 np.save(self.datasetParent.getSpectraFileName(), spectra)
-                
         return spectra
     
     def initializeParticles(self, numParticles):

analysis/particlePainter.py

@@ -142,7 +142,7 @@ class ParticlePainter(QtWidgets.QGraphicsItem):
                 elif y_shift == 0:
                     newImg[:, :self.img.shape[1]] = self.img
                 elif y_shift > 0:
-                    newImg[:y_shift, :self.img.shape[1]] = self.img
+                    newImg[:self.img.shape[0], :self.img.shape[1]] = self.img
             
             self.img = np.uint8(newImg)
             self.setBrect()

analysis/sqlexport.py

 # -*- coding: utf-8 -*-
 """
-Created on Thu Nov  8 13:50:05 2018
-
-@author: brandt
+GEPARD - Gepard-Enabled PARticle Detection
+Copyright (C) 2018  Lars Bittrich and Josef Brandt, Leibniz-Institut für 
+Polymerforschung Dresden e. V. <bittrich-lars@ipfdd.de>    
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program, see COPYING.  
+If not, see <https://www.gnu.org/licenses/>.
 """
 
-from PyQt5 import QtWidgets
+from PyQt5 import QtWidgets, QtCore
 import numpy as np
 import mysql.connector
 import difflib
@@ -14,18 +28,24 @@ import os
 import datetime
 import json
 
+import analysis.DBAssignments as dbassignments
+from analysis.DBAssignments import DBAssignment
+
 class SQLExport(QtWidgets.QDialog):
-    def __init__(self, particleContainer, parent):
-        super(SQLExport, self).__init__(parent)
+    def __init__(self, dataset):
+        super(SQLExport, self).__init__()
         self.setWindowTitle('Export to SQL Database')
         
-        self.particleContainer = particleContainer
-        self.longSizes = self.particleContainer.getSizesOfAllParticles()
-        self.shortSize = self.particleContainer.getShortSizesOfAllParticles()
-        self.spectra = self.particleContainer.spectra
+        self.dataset = dataset
+        self.particleContainer = dataset.particleContainer
+        
+        self.longSizes = np.round(self.particleContainer.getSizesOfAllParticles())
+        self.shortSize = np.round(self.particleContainer.getShortSizesOfAllParticles())
+        self.spectra = self.particleContainer.getSpectraFromDisk()
         
         self.particleImages = None
         self.log = []
+        self.dbAssignments = None
         
         configfilename = os.path.join(os.path.split(__file__)[0], 'database_config.txt')
         
@@ -40,7 +60,7 @@ class SQLExport(QtWidgets.QDialog):
                 for key in ['user', 'password', 'host', 'database', 
                             'raise_on_warnings', 'charset', 'use_unicode']:
                     assert key in self.config, \
-                        'all server config data must be defined in json file'
+                        f'server config key {key} is missing databae_config json file'
         
         self.cnx = None
         self.createInterface()
@@ -49,6 +69,14 @@ class SQLExport(QtWidgets.QDialog):
         layout = QtWidgets.QVBoxLayout()
         self.setLayout(layout)
         
+        assignmentFolder = QtCore.QStandardPaths.writableLocation(QtCore.QStandardPaths.AppLocalDataLocation)
+        assignmentPath = os.path.join(assignmentFolder, 'dbassignments.pkl')
+        if not os.path.exists(assignmentPath):
+            self.dbAssignments = DBAssignment()
+            self.dbAssignments.setFileName(assignmentPath)
+        else:
+            self.dbAssignments = dbassignments.loadAssigments(assignmentPath)
+            
         self.connect()
         
         promptGroup = QtWidgets.QGroupBox('General Properties')
@@ -64,10 +92,10 @@ class SQLExport(QtWidgets.QDialog):
         promptLayout.addWidget(self.sampleEdit, 0, 3)
         promptLayout.addWidget(QtWidgets.QLabel(')'), 0, 4)
         
-        methodList = self.getEntireTable('methods')[1]
-        self.methodIndices = [i[0] for i in methodList]
+        self.methodIndices = self.getMethodIDs()
         self.methodSelector = QtWidgets.QComboBox()
-        self.methodSelector.addItems([i[1] for i in methodList])
+        self.methodSelector.addItems(self.getMethodNames())
+        
         self.methodEdit = LinkedLineEdit(self.methodSelector)
         self.methodEdit.setMaximumWidth(250)
         promptLayout.addWidget(QtWidgets.QLabel('Analysis Method*: '), 1, 0)
@@ -81,6 +109,7 @@ class SQLExport(QtWidgets.QDialog):
         self.analystSelector = QtWidgets.QComboBox()
         self.analystSelector.addItems([', '.join([i[2], i[1]]) for i in analystList])
         self.analysisDate = QtWidgets.QDateEdit()
+        self.analysisDate.setDate(QtCore.QDate().currentDate())
         self.arrivalDate = QtWidgets.QDateEdit()
         furtherProperties = {QtWidgets.QLabel('ArrivalDate*: '): self.arrivalDate,
                              QtWidgets.QLabel('Analysis : '): self.analysisDate, 
@@ -108,37 +137,43 @@ class SQLExport(QtWidgets.QDialog):
 
         self.polymerTypes = self.particleContainer.getUniquePolymers()
         self.polymerCheckboxes = []
-        self.db_polyms = []
-        self.db_categs = []
-        self.p_remarks = []
+        self.db_result_combos = []
+        self.db_categs_combos = []
+        self.db_pRemark_combos = []
         self.comments =[]
         for index, polymerType in enumerate(self.polymerTypes):
             self.polymerCheckboxes.append(QtWidgets.QCheckBox())
-            self.db_polyms.append(QtWidgets.QComboBox())
-            self.db_polyms[-1].addItems([i[1] for i in dbPolymResults])
-            self.setToClosestMatch(polymerType, self.db_polyms[-1])
-            self.db_categs.append(QtWidgets.QComboBox())
-            self.db_categs[-1].addItems([i[1] for i in dbCategResults])
-            self.setToClosestMatch(polymerType, self.db_categs[-1])
-            self.p_remarks.append(QtWidgets.QComboBox())
-            self.p_remarks[-1].addItems([i[1] for i in dbPaintRemark])
-            self.p_remarks[-1].addItem('')
-            self.p_remarks[-1].setCurrentIndex(self.p_remarks[-1].count()-1)
+            newPolymCombo = QtWidgets.QComboBox()
+            newPolymCombo.addItems([i[1] for i in dbPolymResults])
+            self.db_result_combos.append(newPolymCombo)
+            
+            newCategCombo = QtWidgets.QComboBox()
+            newCategCombo.addItems([i[1] for i in dbCategResults])
+            self.db_categs_combos.append(newCategCombo)
+            
+            newPaintRemarksCombo = QtWidgets.QComboBox()
+            newPaintRemarksCombo.addItems([i[1] for i in dbPaintRemark])
+            newPaintRemarksCombo.addItem('')
+            self.db_pRemark_combos.append(newPaintRemarksCombo)
+            
+            self.setComboBoxesToAssignment(newPolymCombo, newCategCombo, newPaintRemarksCombo, polymerType)
+            
+            
             self.comments.append(QtWidgets.QLineEdit())
             
             self.polymerCheckboxes[-1].stateChanged.connect(self.makeStateChangedLambda(self.polymerCheckboxes[-1], 
-                                                                     [self.db_polyms[-1], self.db_categs[-1], self.p_remarks[-1], self.comments[-1]]))
+                                                                     [self.db_result_combos[-1], self.db_categs_combos[-1], self.db_pRemark_combos[-1], self.comments[-1]]))
             self.polymerCheckboxes[-1].setChecked(True)
             self.polymerCheckboxes[-1].setChecked(False)  #switch twice, so that the stateChanged-function is called
             
             typeLayout.addWidget(self.polymerCheckboxes[-1], index+1, 0)
             typeLayout.addWidget(QtWidgets.QLabel(polymerType), index+1, 1)
-            typeLayout.addWidget(self.db_polyms[-1], index+1, 2)
-            typeLayout.addWidget(self.db_categs[-1], index+1, 3)
-            typeLayout.addWidget(self.p_remarks[-1], index+1, 4)
+            typeLayout.addWidget(self.db_result_combos[-1], index+1, 2)
+            typeLayout.addWidget(self.db_categs_combos[-1], index+1, 3)
+            typeLayout.addWidget(self.db_pRemark_combos[-1], index+1, 4)
             typeLayout.addWidget(self.comments[-1], index+1, 5)
         
-        for element in (self.db_polyms + self.db_categs + self.p_remarks + self.comments):
+        for element in (self.db_result_combos + self.db_categs_combos + self.db_pRemark_combos + self.comments):
             element.setMaximumWidth(250)
         
         typeGroupBox.setLayout(typeLayout)
@@ -153,16 +188,6 @@ class SQLExport(QtWidgets.QDialog):