dataset.py 14.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
# -*- coding: utf-8 -*-
"""
GEPARD - Gepard-Enabled PARticle Detection
Copyright (C) 2018  Lars Bittrich and Josef Brandt, Leibniz-Institut für 
Polymerforschung Dresden e. V. <bittrich-lars@ipfdd.de>    

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program, see COPYING.  
If not, see <https://www.gnu.org/licenses/>.
"""
import os
import pickle
import numpy as np
24
import sys
25
import cv2
26
from copy import copy
27 28
from .analysis.particleContainer import ParticleContainer
from .legacyConvert import legacyConversion, currentVersion
29
from .helperfunctions import cv2imwrite_fix, cv2imread_fix
Josef Brandt's avatar
Josef Brandt committed
30

31 32 33
# for legacy pickle import the old module name dataset must be found 
# (no relative import)
from . import dataset
JosefBrandt's avatar
JosefBrandt committed
34
from . import analysis
35
sys.modules['dataset'] = dataset
JosefBrandt's avatar
JosefBrandt committed
36
sys.modules['analysis'] = analysis
Josef Brandt's avatar
Josef Brandt committed
37 38 39
sys.modules['gepardevaluation.dataset'] = dataset
sys.modules['gepardevaluation.analysis.particleContainer'] = analysis.particleContainer
sys.modules['gepardevaluation.analysis.particleAndMeasurement'] = analysis.particleAndMeasurement
40

41

42 43 44 45 46 47 48 49 50
def loadData(fname):
    retds = None
    with open(fname, "rb") as fp:
        ds = pickle.load(fp)
        ds.fname = fname
        ds.readin = True
        retds = DataSet(fname)
        retds.version = 0
        retds.__dict__.update(ds.__dict__)
Josef Brandt's avatar
Josef Brandt committed
51
        retds.updatePath()
52
        if retds.version < currentVersion:
53
            legacyConversion(retds)
54
        elif retds.zvalimg == "saved":
55
            retds.loadZvalImg()
Josef Brandt's avatar
Josef Brandt committed
56 57

        retds.afterLoad()
58 59
    return retds

Josef Brandt's avatar
Josef Brandt committed
60

61
def saveData(dataset, fname):
62
    with open(fname, "wb") as fp:
Josef Brandt's avatar
Josef Brandt committed
63
        # zvalimg is rather large and thus it is saved separately in a tif file
64 65 66 67
        # only onces after its creation
        zvalimg = dataset.zvalimg
        if zvalimg is not None:
            dataset.zvalimg = "saved"
Josef Brandt's avatar
Josef Brandt committed
68 69

        dataset.beforeSave()
70 71
        pickle.dump(dataset, fp, protocol=-1)
        dataset.zvalimg = zvalimg
Josef Brandt's avatar
Josef Brandt committed
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
        dataset.afterSave()


# def loadData(fname):
#     retds = None
#     with open(fname, "rb") as fp:
#         ds = pickle.load(fp)
#         ds.fname = fname
#         ds.readin = True
#         ds.updatePath()
#         retds = DataSet(fname)
#         retds.version = 0
#         retds.__dict__.update(ds.__dict__)
#         if retds.version < currentVersion:
#             legacyConversion(retds)
#         elif retds.zvalimg == "saved":
#             retds.loadZvalImg()
#         retds.particleContainer.datasetParent = retds #TODO: This is mainly a workaround to update the ref in particleContainer. It probably should be handled differently anyways...
#     return retds
#
# def saveData(dataset, fname):
#     with open(fname, "wb") as fp:
#         # zvalimg is rather large and thus it is saved separately in a tif file
#         # only onces after its creation
#         zvalimg = dataset.zvalimg
#         if zvalimg is not None:
#             dataset.zvalimg = "saved"
#         pickle.dump(dataset, fp, protocol=-1)
#         dataset.zvalimg = zvalimg
101

102
def arrayCompare(a1, a2):
Josef Brandt's avatar
Josef Brandt committed
103
    if a1.shape != a2.shape:
Lars Bittrich's avatar
Lars Bittrich committed
104
        return False
Josef Brandt's avatar
Josef Brandt committed
105 106
    if a1.dtype != np.float32 and a1.dtype != np.float64:
        return np.all(a1 == a2)
107 108
    ind = np.isnan(a1)
    if not np.any(ind):
Josef Brandt's avatar
Josef Brandt committed
109 110 111
        return np.all(a1 == a2)
    return np.all(a1[~ind] == a2[~ind])

112 113

def listCompare(l1, l2):
Josef Brandt's avatar
Josef Brandt committed
114
    if len(l1) != len(l2):
115 116 117 118 119 120 121 122
        return False
    for l1i, l2i in zip(l1, l2):
        if isinstance(l1i, np.ndarray):
            if not isinstance(l2i, np.ndarray) or not arrayCompare(l1i, l2i):
                return False
        elif isinstance(l1i, (list, tuple)):
            if not isinstance(l2i, (list, tuple)) or not listCompare(l1i, l2i):
                return False
Josef Brandt's avatar
Josef Brandt committed
123
        elif l1i != l2i and ((~np.isnan(l1i)) or (~np.isnan(l2i))):
124 125 126
            return False
    return True

Josef Brandt's avatar
Josef Brandt committed
127

128 129 130
def recursiveDictCompare(d1, d2):
    for key in d1:
        if not key in d2:
Lars Bittrich's avatar
Lars Bittrich committed
131
            print("key missing in d2:", key, flush=True)
132 133 134
            return False
        a = d1[key]
        b = d2[key]
Lars Bittrich's avatar
Lars Bittrich committed
135
        print(key, type(a), type(b), flush=True)
136 137
        if isinstance(a, np.ndarray):
            if not isinstance(b, np.ndarray) or not arrayCompare(a, b):
Lars Bittrich's avatar
Lars Bittrich committed
138
                print("data is different!", a, b)
139 140 141
                return False
        elif isinstance(a, dict):
            if not isinstance(b, dict):
Lars Bittrich's avatar
Lars Bittrich committed
142
                print("data is different!", a, b)
143 144 145 146 147
                return False
            if not recursiveDictCompare(a, b):
                return False
        elif isinstance(a, (list, tuple)):
            if not isinstance(b, (list, tuple)) or not listCompare(a, b):
Lars Bittrich's avatar
Lars Bittrich committed
148
                print("data is different!", a, b)
149 150
                return False
        elif a != b:
Lars Bittrich's avatar
Lars Bittrich committed
151 152 153
            if (a is not None) and (b is not None):
                print("data is different!", a, b)
                return False
154 155
    return True

156

157
class DataSet(object):
Josef Brandt's avatar
Josef Brandt committed
158 159 160 161 162 163 164 165 166 167 168
    uid: int = 0

    @classmethod
    def getUID(cls):
        """
        returns a unique number managed by the dataset. will be saved and restored on load
        :return int:
        """
        cls.uid += 1
        return cls.uid

169 170 171
    def __init__(self, fname, newProject=False):
        self.fname = fname
        # parameters specifically for optical scan
172
        self.version = currentVersion
173 174
        self.lastpos = None
        self.maxdim = None
175 176
        self.pixelscale_df = None  # µm / pixel --> scale of DARK FIELD camera (used for image stitching)
        self.pixelscale_bf = None  # µm / pixel of DARK FIELD camera (set to same as bright field, if both use the same camera)
177 178
        self.imagedim_bf = None  # width, height, angle of BRIGHT FIELD camera
        self.imagedim_df = None  # width, height, angle of DARK FIELD camera (set to same as bright field, if both use the same camera)
179
        self.imagescanMode = 'df'  # was the fullimage acquired in dark- or brightfield?
Josef Brandt's avatar
Josef Brandt committed
180
        self.fitpoints = []  # manually adjusted positions aquired to define the specimen geometry
181
        self.fitindices = []  # which of the five positions in the ui are already known
Josef Brandt's avatar
Josef Brandt committed
182 183
        self.boundary = []  # scan boundary computed by a circle around the fitpoints + manual adjustments
        self.grid = []  # scan grid positions for optical scan
184 185 186
        self.zpositions = []  # z-positions for optical scan
        self.heightmap = None
        self.zvalimg = None
187
        self.coordinatetransform = None  # if imported form extern source coordinate system may be rotated
188 189
        self.signx = 1.
        self.signy = -1.
190 191 192

        # tiling parameters
        self.pyramidParams = None
Josef Brandt's avatar
Josef Brandt committed
193 194 195 196

        # navigation params (e.g. visited particles)
        self.particleNavigationParams = None

197
        # parameters specifically for raman scan
Josef Brandt's avatar
Josef Brandt committed
198
        self.pshift = None  # shift of spectrometer scan position relative to image center
199 200
        self.seedpoints = np.array([])
        self.seeddeletepoints = np.array([])
Josef Brandt's avatar
Josef Brandt committed
201
        self.detectParams = {'points': np.array([[50, 0], [100, 200], [200, 255]]),
Josef Brandt's avatar
Josef Brandt committed
202 203 204 205 206 207 208
                             'contrastcurve': True,
                             'blurRadius': 9,
                             'threshold': 0.2,
                             'maxholebrightness': 0.5,
                             'minparticlearea': 20,
                             'minparticledistance': 20,
                             'measurefrac': 1,
209
                             'compactness': 0.0,
Josef Brandt's avatar
Josef Brandt committed
210
                             'seedRad': 3}
JosefBrandt's avatar
JosefBrandt committed
211

Josef Brandt's avatar
Josef Brandt committed
212
        self.particleContainer = ParticleContainer()
213
        self.particleDetectionDone = False
214
        self.specscandone = False
215

JosefBrandt's avatar
JosefBrandt committed
216
        self.resultParams = {'minHQI': 5}
217 218
        self.colorSeed = 'default'
        self.resultsUploadedToSQL = []
Josef Brandt's avatar
Josef Brandt committed
219 220 221

        self.readin = True  # a value that is always set to True at loadData
        # and mark that the coordinate system might be changed in the meantime
222 223 224 225
        self.mode = "prepare"
        if newProject:
            self.fname = self.newProject(fname)
        self.updatePath()
Josef Brandt's avatar
Josef Brandt committed
226 227 228 229 230

    @property
    def opticalScanDone(self) -> bool:
        return os.path.exists(self.getZvalImageName())

231 232
    def __eq__(self, other):
        return recursiveDictCompare(self.__dict__, other.__dict__)
233

Josef Brandt's avatar
Josef Brandt committed
234 235 236 237 238 239 240 241 242 243 244 245
    def afterLoad(self):
        self.particleContainer.datasetParent = self
        self.updatePath()
        DataSet.uid = self.uid

    def beforeSave(self):
        self.particleContainer.datasetParent = None
        self.uid = DataSet.uid

    def afterSave(self):
        self.afterLoad()

246 247 248 249 250
    def getPyramidParams(self):
        return self.pyramidParams

    def setPyramidParams(self, pyramid_params):
        self.pyramidParams = pyramid_params
Josef Brandt's avatar
Josef Brandt committed
251 252 253 254 255 256 257

    def getParticleNavigationParams(self):
        return self.particleNavigationParams

    def setParticleNavigationParams(self, part_nav_params):
        self.particleNavigationParams = part_nav_params

258 259 260
    def getPixelScale(self, mode=None):
        if mode is None:
            mode = self.imagescanMode
261
        return (self.pixelscale_df if mode == "df" else self.pixelscale_bf)
Josef Brandt's avatar
Josef Brandt committed
262

263 264 265
    def saveZvalImg(self):
        if self.zvalimg is not None:
            cv2imwrite_fix(self.getZvalImageName(), self.zvalimg)
266
            self.zvalimg = "saved"
267 268 269 270

    def loadZvalImg(self):
        if os.path.exists(self.getZvalImageName()):
            self.zvalimg = cv2imread_fix(self.getZvalImageName(), cv2.IMREAD_GRAYSCALE)
Josef Brandt's avatar
Josef Brandt committed
271

272 273
    def getZval(self, pixelpos):
        assert self.zvalimg is not None
274 275
        if self.zvalimg == "saved":
            self.loadZvalImg()
276
        i, j = int(round(pixelpos[1])), int(round(pixelpos[0]))
Josef Brandt's avatar
Josef Brandt committed
277
        if i >= self.zvalimg.shape[0]:
278
            print('error in getZval:', self.zvalimg.shape, i, j)
Josef Brandt's avatar
Josef Brandt committed
279 280
            i = self.zvalimg.shape[0] - 1
        if j >= self.zvalimg.shape[1]:
281
            print('error in getZval:', self.zvalimg.shape, i, j)
Josef Brandt's avatar
Josef Brandt committed
282 283
            j = self.zvalimg.shape[1] - 1
        zp = self.zvalimg[i, j]
284
        z0, z1 = self.zpositions[0], self.zpositions[-1]
Josef Brandt's avatar
Josef Brandt committed
285 286
        return zp / 255. * (z1 - z0) + z0

287 288 289
    def mapHeight(self, x, y, force=False):
        if not force:
            assert not self.readin
290
        assert self.heightmap is not None
Josef Brandt's avatar
Josef Brandt committed
291 292
        return self.heightmap[0] * x + self.heightmap[1] * y + self.heightmap[2]

293
    def mapToPixel(self, p, mode='df', force=False):
294 295 296
        if not force:
            assert not self.readin
        p0 = copy(self.lastpos)
Josef Brandt's avatar
Josef Brandt committed
297

298
        if self.coordinatetransform is not None:
Josef Brandt's avatar
Josef Brandt committed
299
            z = 0. if len(p) < 3 else p[2]
300
            T, pc = self.coordinatetransform
Josef Brandt's avatar
Josef Brandt committed
301
            p = (np.dot(np.array([p[0], p[1], z]) - pc, T.T))
Josef Brandt's avatar
Josef Brandt committed
302 303 304

        assert mode in ['df', 'bf'], f'mapToPixel mode: {mode} not understood'
        pixelscale: float = self.pixelscale_bf if mode == 'bf' else self.pixelscale_df
Josef Brandt's avatar
Josef Brandt committed
305 306 307
        p0[0] -= self.signx * self.imagedim_df[0] / 2
        p0[1] -= self.signy * self.imagedim_df[1] / 2
        x, y = self.signx * (p[0] - p0[0]) / pixelscale, self.signy * (p[1] - p0[1]) / pixelscale
308
        return x, y
Josef Brandt's avatar
Josef Brandt committed
309

310
    def mapToLength(self, pixelpos, mode='df', force=False, returnz=False):
311 312 313
        if not force:
            assert not self.readin
        p0 = copy(self.lastpos)
314
        if mode == 'df':
Josef Brandt's avatar
Josef Brandt committed
315 316 317 318
            p0[0] -= self.signx * self.imagedim_df[0] / 2
            p0[1] -= self.signy * self.imagedim_df[1] / 2
            x, y = (self.signx * pixelpos[0] * self.pixelscale_df + p0[0]), (
                        p0[1] + self.signy * pixelpos[1] * self.pixelscale_df)
319
        elif mode == 'bf':
Josef Brandt's avatar
Josef Brandt committed
320 321 322 323
            p0[0] -= self.signx * self.imagedim_bf[0] / 2
            p0[1] -= self.signy * self.imagedim_bf[1] / 2
            x, y = (self.signx * pixelpos[0] * self.pixelscale_bf + p0[0]), (
                        p0[1] + self.signy * pixelpos[1] * self.pixelscale_bf)
324
        else:
325
            raise ValueError(f'mapToLength mode: {mode} not understood')
Josef Brandt's avatar
Josef Brandt committed
326

327
        z = None
328
        if (returnz and self.zvalimg is not None) or self.coordinatetransform is not None:
329
            z = self.mapHeight(x, y, force=force)
330
            z += self.getZval(pixelpos)
Josef Brandt's avatar
Josef Brandt committed
331

332 333
        if self.coordinatetransform is not None:
            T, pc = self.coordinatetransform
334
            x, y, z = (np.dot(np.array([x, y, z]), T) + pc)
Josef Brandt's avatar
Josef Brandt committed
335

336 337 338
        if returnz:
            return x, y, z
        return x, y
Josef Brandt's avatar
Josef Brandt committed
339

340
    def mapToLengthSpectrometer(self, pixelpos, microscopeMode='df', noz=False):
341 342
        p0x, p0y, z = self.mapToLength(pixelpos, mode=microscopeMode, returnz=True)
        x, y = p0x + self.pshift[0], p0y + self.pshift[1]
343
        return x, y, z
Josef Brandt's avatar
Josef Brandt committed
344

345 346 347 348 349 350
    def newProject(self, fname):
        path = os.path.split(fname)[0]
        name = os.path.splitext(os.path.basename(fname))[0]
        newpath = os.path.join(path, name)
        fname = os.path.join(newpath, name + ".pkl")
        if not os.path.exists(newpath):
Josef Brandt's avatar
Josef Brandt committed
351
            os.mkdir(newpath)  # for new projects a directory will be created
352 353 354
        elif os.path.exists(fname):  # if this project is already there, load it instead
            self.__dict__.update(loadData(fname).__dict__)
        return fname
Josef Brandt's avatar
Josef Brandt committed
355

356 357 358 359 360 361
    def getTilePath(self):
        scandir = os.path.join(self.path, "tiles")
        if not os.path.exists(scandir):
            os.mkdir(scandir)
        return scandir

362 363 364 365 366
    def getScanPath(self):
        scandir = os.path.join(self.path, "scanimages")
        if not os.path.exists(scandir):
            os.mkdir(scandir)
        return scandir
Josef Brandt's avatar
Josef Brandt committed
367

368 369 370
    def updatePath(self):
        self.path = os.path.split(self.fname)[0]
        self.name = os.path.splitext(os.path.basename(self.fname))[0]
Josef Brandt's avatar
Josef Brandt committed
371

JosefBrandt's avatar
JosefBrandt committed
372
    def getSpectraFileName(self):
JosefBrandt's avatar
 
JosefBrandt committed
373
        return os.path.join(self.path, 'spectra.npy')
Josef Brandt's avatar
Josef Brandt committed
374

375
    def getImageName(self):
376 377
        return os.path.join(self.path, 'fullimage.tif')

378 379
    def getZvalImageName(self):
        return os.path.join(self.path, "zvalues.tif")
Josef Brandt's avatar
Josef Brandt committed
380

381 382
    def getLegacyImageName(self):
        return os.path.join(self.path, "fullimage.png")
Josef Brandt's avatar
Josef Brandt committed
383

Josef Brandt's avatar
 
Josef Brandt committed
384 385
    def getBackgroundImageName(self):
        return os.path.join(self.path, "background.bmp")
Josef Brandt's avatar
Josef Brandt committed
386

387 388
    def getTmpImageName(self):
        return os.path.join(self.path, "tmp.bmp")
Josef Brandt's avatar
Josef Brandt committed
389

390 391
    def save(self):
        saveData(self, self.fname)
Josef Brandt's avatar
Josef Brandt committed
392

393 394
    def saveBackup(self):
        inc = 0
Hackmet's avatar
Hackmet committed
395
        while True:
396 397 398 399 400 401 402
            directory = os.path.dirname(self.fname)
            filename = self.name + '_backup_' + str(inc) + '.pkl'
            path = os.path.join(directory, filename)
            if os.path.exists(path):
                inc += 1
            else:
                saveData(self, path)
Hackmet's avatar
Hackmet committed
403
                return filename