dataset.py 12.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
# -*- coding: utf-8 -*-
"""
GEPARD - Gepard-Enabled PARticle Detection
Copyright (C) 2018  Lars Bittrich and Josef Brandt, Leibniz-Institut für 
Polymerforschung Dresden e. V. <bittrich-lars@ipfdd.de>    

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program, see COPYING.  
If not, see <https://www.gnu.org/licenses/>.
"""
import os
import pickle
import numpy as np
24
import sys
25
import cv2
26
from copy import copy
27 28
from .analysis.particleContainer import ParticleContainer
from .legacyConvert import legacyConversion, currentVersion
29
from .helperfunctions import cv2imwrite_fix, cv2imread_fix
30 31 32
# for legacy pickle import the old module name dataset must be found 
# (no relative import)
from . import dataset
JosefBrandt's avatar
JosefBrandt committed
33
from . import analysis
34
sys.modules['dataset'] = dataset
JosefBrandt's avatar
JosefBrandt committed
35
sys.modules['analysis'] = analysis
36

37

38 39 40 41 42 43 44 45 46 47
def loadData(fname):
    retds = None
    with open(fname, "rb") as fp:
        ds = pickle.load(fp)
        ds.fname = fname
        ds.readin = True
        ds.updatePath()
        retds = DataSet(fname)
        retds.version = 0
        retds.__dict__.update(ds.__dict__)
48
        if retds.version < currentVersion:
49
            legacyConversion(retds)
50
        elif retds.zvalimg == "saved":
51
            retds.loadZvalImg()
52
        retds.particleContainer.datasetParent = retds #TODO: This is mainly a workaround to update the ref in particleContainer. It probably should be handled differently anyways...
53 54 55
    return retds

def saveData(dataset, fname):
56 57 58 59 60 61 62 63
    with open(fname, "wb") as fp:
        # zvalimg is rather large and thus it is saved separately in a tif file 
        # only onces after its creation
        zvalimg = dataset.zvalimg
        if zvalimg is not None:
            dataset.zvalimg = "saved"
        pickle.dump(dataset, fp, protocol=-1)
        dataset.zvalimg = zvalimg
64

65
def arrayCompare(a1, a2):
Lars Bittrich's avatar
Lars Bittrich committed
66 67 68 69
    if a1.shape!=a2.shape:
        return False
    if a1.dtype!=np.float32 and a1.dtype!=np.float64:
        return np.all(a1==a2)
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
    ind = np.isnan(a1)
    if not np.any(ind):
        return np.all(a1==a2)
    return np.all(a1[~ind]==a2[~ind])

def listCompare(l1, l2):
    if len(l1)!=len(l2):
        return False
    for l1i, l2i in zip(l1, l2):
        if isinstance(l1i, np.ndarray):
            if not isinstance(l2i, np.ndarray) or not arrayCompare(l1i, l2i):
                return False
        elif isinstance(l1i, (list, tuple)):
            if not isinstance(l2i, (list, tuple)) or not listCompare(l1i, l2i):
                return False
        elif l1i!=l2i and ((~np.isnan(l1i)) or (~np.isnan(l2i))):
            return False
    return True

def recursiveDictCompare(d1, d2):
    for key in d1:
        if not key in d2:
Lars Bittrich's avatar
Lars Bittrich committed
92
            print("key missing in d2:", key, flush=True)
93 94 95
            return False
        a = d1[key]
        b = d2[key]
Lars Bittrich's avatar
Lars Bittrich committed
96
        print(key, type(a), type(b), flush=True)
97 98
        if isinstance(a, np.ndarray):
            if not isinstance(b, np.ndarray) or not arrayCompare(a, b):
Lars Bittrich's avatar
Lars Bittrich committed
99
                print("data is different!", a, b)
100 101 102
                return False
        elif isinstance(a, dict):
            if not isinstance(b, dict):
Lars Bittrich's avatar
Lars Bittrich committed
103
                print("data is different!", a, b)
104 105 106 107 108
                return False
            if not recursiveDictCompare(a, b):
                return False
        elif isinstance(a, (list, tuple)):
            if not isinstance(b, (list, tuple)) or not listCompare(a, b):
Lars Bittrich's avatar
Lars Bittrich committed
109
                print("data is different!", a, b)
110 111
                return False
        elif a != b:
Lars Bittrich's avatar
Lars Bittrich committed
112 113 114
            if (a is not None) and (b is not None):
                print("data is different!", a, b)
                return False
115 116
    return True

117

118 119 120 121
class DataSet(object):
    def __init__(self, fname, newProject=False):
        self.fname = fname
        # parameters specifically for optical scan
122
        self.version = currentVersion
123 124
        self.lastpos = None
        self.maxdim = None
125 126
        self.pixelscale_df = None  # µm / pixel --> scale of DARK FIELD camera (used for image stitching)
        self.pixelscale_bf = None  # µm / pixel of DARK FIELD camera (set to same as bright field, if both use the same camera)
127 128
        self.imagedim_bf = None  # width, height, angle of BRIGHT FIELD camera
        self.imagedim_df = None  # width, height, angle of DARK FIELD camera (set to same as bright field, if both use the same camera)
129
        self.imagescanMode = 'df'  # was the fullimage acquired in dark- or brightfield?
130 131 132 133 134 135 136
        self.fitpoints = []   # manually adjusted positions aquired to define the specimen geometry
        self.fitindices = []  # which of the five positions in the ui are already known
        self.boundary = []    # scan boundary computed by a circle around the fitpoints + manual adjustments
        self.grid = []        # scan grid positions for optical scan
        self.zpositions = []  # z-positions for optical scan
        self.heightmap = None
        self.zvalimg = None
137
        self.coordinatetransform = None  # if imported form extern source coordinate system may be rotated
138 139
        self.signx = 1.
        self.signy = -1.
140 141 142

        # tiling parameters
        self.pyramidParams = None
143 144
        
        # parameters specifically for raman scan
145
        self.pshift = None    # shift of spectrometer scan position relative to image center
146 147
        self.seedpoints = np.array([])
        self.seeddeletepoints = np.array([])
Josef Brandt's avatar
Josef Brandt committed
148 149 150 151 152 153 154 155
        self.detectParams = {'points': np.array([[50,0],[100,200],[200,255]]),
                             'contrastcurve': True,
                             'blurRadius': 9,
                             'threshold': 0.2,
                             'maxholebrightness': 0.5,
                             'minparticlearea': 20,
                             'minparticledistance': 20,
                             'measurefrac': 1,
156
                             'compactness': 0.0,
Josef Brandt's avatar
Josef Brandt committed
157
                             'seedRad': 3}
JosefBrandt's avatar
JosefBrandt committed
158

JosefBrandt's avatar
JosefBrandt committed
159
        self.particleContainer = ParticleContainer(self)
160
        self.particleDetectionDone = False
161
        self.specscandone = False
162

JosefBrandt's avatar
JosefBrandt committed
163
        self.resultParams = {'minHQI': 5}
164 165 166
        self.colorSeed = 'default'
        self.resultsUploadedToSQL = []
        
167 168 169 170 171 172
        self.readin = True    # a value that is always set to True at loadData 
                              # and mark that the coordinate system might be changed in the meantime
        self.mode = "prepare"
        if newProject:
            self.fname = self.newProject(fname)
        self.updatePath()
Josef Brandt's avatar
Josef Brandt committed
173 174 175 176 177

    @property
    def opticalScanDone(self) -> bool:
        return os.path.exists(self.getZvalImageName())

178 179
    def __eq__(self, other):
        return recursiveDictCompare(self.__dict__, other.__dict__)
180 181 182 183 184 185

    def getPyramidParams(self):
        return self.pyramidParams

    def setPyramidParams(self, pyramid_params):
        self.pyramidParams = pyramid_params
186 187 188 189
        
    def getPixelScale(self, mode=None):
        if mode is None:
            mode = self.imagescanMode
190
        return (self.pixelscale_df if mode == "df" else self.pixelscale_bf)
191
    
192 193 194
    def saveZvalImg(self):
        if self.zvalimg is not None:
            cv2imwrite_fix(self.getZvalImageName(), self.zvalimg)
195
            self.zvalimg = "saved"
196 197 198 199 200

    def loadZvalImg(self):
        if os.path.exists(self.getZvalImageName()):
            self.zvalimg = cv2imread_fix(self.getZvalImageName(), cv2.IMREAD_GRAYSCALE)
    
201 202
    def getZval(self, pixelpos):
        assert self.zvalimg is not None
203 204
        if self.zvalimg == "saved":
            self.loadZvalImg()
205 206 207 208 209 210 211 212 213
        i, j = int(round(pixelpos[1])), int(round(pixelpos[0]))
        if i>=self.zvalimg.shape[0]: 
            print('error in getZval:', self.zvalimg.shape, i, j)
            i = self.zvalimg.shape[0]-1
        if j>=self.zvalimg.shape[1]: 
            print('error in getZval:', self.zvalimg.shape, i, j)
            j = self.zvalimg.shape[1]-1
        zp = self.zvalimg[i,j]
        z0, z1 = self.zpositions[0], self.zpositions[-1]
214
        return zp/255.*(z1-z0) + z0
215
    
216 217 218 219 220
    def mapHeight(self, x, y):
        assert not self.readin
        assert self.heightmap is not None
        return self.heightmap[0]*x + self.heightmap[1]*y + self.heightmap[2]
        
221
    def mapToPixel(self, p, mode='df', force=False):
222 223 224
        if not force:
            assert not self.readin
        p0 = copy(self.lastpos)
225
        
226
        if self.coordinatetransform is not None:
Josef Brandt's avatar
Josef Brandt committed
227
            z = 0. if len(p) < 3 else p[2]
228 229
            T, pc = self.coordinatetransform
            p = (np.dot(np.array([p[0], p[1], z])-pc, T.T))
Josef Brandt's avatar
Josef Brandt committed
230 231 232 233 234 235

        assert mode in ['df', 'bf'], f'mapToPixel mode: {mode} not understood'
        pixelscale: float = self.pixelscale_bf if mode == 'bf' else self.pixelscale_df
        p0[0] -= self.signx*self.imagedim_df[0]/2
        p0[1] -= self.signy*self.imagedim_df[1]/2
        x, y = self.signx*(p[0] - p0[0])/pixelscale, self.signy*(p[1] - p0[1])/pixelscale
236 237 238
        return x, y
            
    def mapToLength(self, pixelpos, mode='df', force=False, returnz=False):
239 240 241
        if not force:
            assert not self.readin
        p0 = copy(self.lastpos)
242
        if mode == 'df':
243 244 245
            p0[0] -= self.signx*self.imagedim_df[0]/2
            p0[1] -= self.signy*self.imagedim_df[1]/2
            x, y = (self.signx*pixelpos[0]*self.pixelscale_df + p0[0]), (p0[1] + self.signy*pixelpos[1]*self.pixelscale_df)
246
        elif mode == 'bf':
247 248 249
            p0[0] -= self.signx*self.imagedim_bf[0]/2
            p0[1] -= self.signy*self.imagedim_bf[1]/2
            x, y = (self.signx*pixelpos[0]*self.pixelscale_bf + p0[0]), (p0[1] + self.signy*pixelpos[1]*self.pixelscale_bf)
250
        else:
251
            raise ValueError(f'mapToLength mode: {mode} not understood')
252
        
253
        z = None
254
        if (returnz and self.zvalimg is not None) or self.coordinatetransform is not None:
255 256
            z = self.mapHeight(x, y)
            z += self.getZval(pixelpos)
257 258 259
            
        if self.coordinatetransform is not None:
            T, pc = self.coordinatetransform
260
            x, y, z = (np.dot(np.array([x, y, z]), T) + pc)
261 262 263 264 265
        
        if returnz:
            return x, y, z
        return x, y
    
266
    def mapToLengthSpectrometer(self, pixelpos, microscopeMode='df', noz=False):
267 268
        p0x, p0y, z = self.mapToLength(pixelpos, mode=microscopeMode, returnz=True)
        x, y = p0x + self.pshift[0], p0y + self.pshift[1]
269 270 271 272 273 274 275 276 277 278 279 280 281
        return x, y, z
        
    def newProject(self, fname):
        path = os.path.split(fname)[0]
        name = os.path.splitext(os.path.basename(fname))[0]
        newpath = os.path.join(path, name)
        fname = os.path.join(newpath, name + ".pkl")
        if not os.path.exists(newpath):
            os.mkdir(newpath)        # for new projects a directory will be created
        elif os.path.exists(fname):  # if this project is already there, load it instead
            self.__dict__.update(loadData(fname).__dict__)
        return fname
    
282 283 284 285 286 287
    def getTilePath(self):
        scandir = os.path.join(self.path, "tiles")
        if not os.path.exists(scandir):
            os.mkdir(scandir)
        return scandir

288 289 290 291 292 293 294 295 296
    def getScanPath(self):
        scandir = os.path.join(self.path, "scanimages")
        if not os.path.exists(scandir):
            os.mkdir(scandir)
        return scandir
        
    def updatePath(self):
        self.path = os.path.split(self.fname)[0]
        self.name = os.path.splitext(os.path.basename(self.fname))[0]
JosefBrandt's avatar
JosefBrandt committed
297 298
    
    def getSpectraFileName(self):
JosefBrandt's avatar
 
JosefBrandt committed
299
        return os.path.join(self.path, 'spectra.npy')
JosefBrandt's avatar
JosefBrandt committed
300
    
301
    def getImageName(self):
302 303
        return os.path.join(self.path, 'fullimage.tif')

304 305 306 307 308 309
    def getZvalImageName(self):
        return os.path.join(self.path, "zvalues.tif")
    
    def getLegacyImageName(self):
        return os.path.join(self.path, "fullimage.png")
    
Josef Brandt's avatar
 
Josef Brandt committed
310 311 312
    def getBackgroundImageName(self):
        return os.path.join(self.path, "background.bmp")
    
313 314 315 316 317
    def getTmpImageName(self):
        return os.path.join(self.path, "tmp.bmp")
            
    def save(self):
        saveData(self, self.fname)
318 319 320
    
    def saveBackup(self):
        inc = 0
Hackmet's avatar
Hackmet committed
321
        while True:
322 323 324 325 326 327 328
            directory = os.path.dirname(self.fname)
            filename = self.name + '_backup_' + str(inc) + '.pkl'
            path = os.path.join(directory, filename)
            if os.path.exists(path):
                inc += 1
            else:
                saveData(self, path)
Hackmet's avatar
Hackmet committed
329 330
                return filename