test_evaluation.py 17.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 22 13:58:25 2020

@author: luna
"""

import unittest
import random
import sys
sys.path.append("C://Users//xbrjos//Desktop//Python")
import gepard
from gepard.analysis.particleAndMeasurement import Particle, Measurement

Josef Brandt's avatar
Josef Brandt committed
16 17 18
from evaluation import TotalResults, SampleResult, SubsamplingResult
import methods as meth
import geometricMethods as gmeth
19

Josef Brandt's avatar
Josef Brandt committed
20 21 22 23 24 25

class TestTotalResults(unittest.TestCase):
    def setUp(self) -> None:
        self.totalResults = TotalResults()

    def test_add_sample(self):
26
        newResult: SampleResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile.pkl')
Josef Brandt's avatar
Josef Brandt committed
27
        self.assertEqual(len(self.totalResults.sampleResults), 1)
28
        self.assertTrue(type(newResult) == SampleResult)
Josef Brandt's avatar
Josef Brandt committed
29

30
        newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile.pkl')  # the same file should not be added again
Josef Brandt's avatar
Josef Brandt committed
31
        self.assertEqual(len(self.totalResults.sampleResults), 1)
32
        self.assertTrue(newResult is None)
Josef Brandt's avatar
Josef Brandt committed
33

34
        newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile2.pkl')  # another should be added, though
Josef Brandt's avatar
Josef Brandt committed
35
        self.assertEqual(len(self.totalResults.sampleResults), 2)
36
        self.assertTrue(type(newResult) == SampleResult)
Josef Brandt's avatar
Josef Brandt committed
37

38
        newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile2.txt')  # invalid extention, not added...
Josef Brandt's avatar
Josef Brandt committed
39
        self.assertEqual(len(self.totalResults.sampleResults), 2)
40
        self.assertTrue(newResult is None)
Josef Brandt's avatar
Josef Brandt committed
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95

    def test_get_methods_for_fraction(self):
        def containsMethod(listOfMethods: list, template: meth.SubsamplingMethod) -> bool:
            contains: bool = False
            for method in listOfMethods:
                if type(method) == type(template) and method.fraction == template.fraction:
                    contains = True
                    break
            return contains

        dset: gepard.dataset.DataSet = gepard.dataset.DataSet('fakepath')

        imgdim = 10
        dset.imagescanMode = 'df'
        dset.imagedim_df = [imgdim, imgdim]
        dset.pixelscale_df = 1.0
        minX, maxX, minY, maxY = 0, 1000, 0, 1000
        dset.maxdim = minX + imgdim / 2, maxY - imgdim / 2, maxX - imgdim / 2, minY + imgdim / 2

        desiredFraction = 0.1
        methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
        possibleRandomMethods = 2
        possibleCrossBoxMethods = 2
        possibleSpiralBoxMethods = 3
        totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
        self.assertEqual(len(methods), totalPossible)
        self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))

        desiredFraction = 0.5
        methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
        possibleRandomMethods = 2
        possibleCrossBoxMethods = 1
        possibleSpiralBoxMethods = 0
        totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
        self.assertEqual(len(methods), totalPossible)
        self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
        self.assertFalse(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))

        desiredFraction = 0.9
        methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
        possibleRandomMethods = 2
        possibleCrossBoxMethods = 0
        possibleSpiralBoxMethods = 0
        totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
        self.assertEqual(len(methods), totalPossible)
        self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
        self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
        self.assertFalse(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
        self.assertFalse(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))

96
    def test_get_error_vs_fraction_data(self):
97
        firstSample: SampleResult = self.totalResults.add_sample('sample1.pkl')
98
        firstSample.set_attribute('to be used')
99
        secondSample: SampleResult = self.totalResults.add_sample('sample2.pkl')
100
        secondSample.set_attribute('not to be used')
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145

        firstMethod: meth.RandomSampling = meth.RandomSampling(None, 0.1)
        firstResult: SubsamplingResult = SubsamplingResult(firstMethod)
        firstResult.mpCountError = 0.8

        secondMethod: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
        secondMethod.numBoxesAcross = 3
        secondResult: SubsamplingResult = SubsamplingResult(secondMethod)
        secondResult.mpCountError = 0.6

        thirdMethod: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
        thirdMethod.numBoxesAcross = 5
        self.assertEqual(thirdMethod.fraction, 0.1)
        thirdResult: SubsamplingResult = SubsamplingResult(thirdMethod)
        thirdResult.mpCountError = 0.4

        thirdMethod2: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
        thirdMethod2.numBoxesAcross = 5
        self.assertEqual(thirdMethod2.fraction, 0.1)
        thirdResult2: SubsamplingResult = SubsamplingResult(thirdMethod)
        thirdResult2.mpCountError = 0.8

        thirdMethod3: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.2)
        thirdMethod3.numBoxesAcross = 5
        self.assertEqual(thirdMethod3.fraction, 0.2)
        thirdResult3: SubsamplingResult = SubsamplingResult(thirdMethod3)
        thirdResult3.mpCountError = 0.5

        firstSample.results = [firstResult, secondResult, thirdResult, thirdResult3]
        secondSample.results = [firstResult, secondResult, thirdResult2, thirdResult3]

        resultDict: dict = self.totalResults.get_error_vs_fraction_data()
        self.assertEqual(list(resultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
        for i in range(3):
            res: dict = list(resultDict.values())[i]
            if i == 0:
                self.assertEqual(list(res.keys()), [0.1])
                self.assertAlmostEqual(res[0.1], 0.8)
            if i == 1:
                self.assertEqual(list(res.keys()), [0.1])
                self.assertAlmostEqual(res[0.1], 0.6)
            if i == 2:
                self.assertEqual(list(res.keys()), [0.1, 0.2])
                self.assertAlmostEqual(res[0.1], 0.6)  # i.e., mean([0.4, 0.8])
                self.assertAlmostEqual(res[0.2], 0.5)
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169

        filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(attributes=['to be used'])
        self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
        for i in range(3):
            res: dict = list(filteredResultDict.values())[i]
            if i == 0:
                self.assertEqual(list(res.keys()), [0.1])
                self.assertAlmostEqual(res[0.1], 0.8)
            if i == 1:
                self.assertEqual(list(res.keys()), [0.1])
                self.assertAlmostEqual(res[0.1], 0.6)
            if i == 2:
                self.assertEqual(list(res.keys()), [0.1, 0.2])
                self.assertAlmostEqual(res[0.1], 0.4)  # only the result from the first sample is used, as filtered..
                self.assertAlmostEqual(res[0.2], 0.5)

        filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['cross'])
        self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])

        filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['Cross'])
        self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])

        filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['random'])
        self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label])
170

Josef Brandt's avatar
Josef Brandt committed
171 172 173 174 175 176

class TestSampleResult(unittest.TestCase):
    def setUp(self) -> None:
        self.sampleResult: SampleResult = SampleResult('fakePath/fakeFile.pkl')
        self.sampleResult.dataset = gepard.dataset.DataSet('fakePath/fakeFile.pkl')
        self.sampleResult.results.append(SubsamplingResult(meth.RandomSampling(None, 0.1)))
177 178 179 180 181 182 183 184 185 186 187 188

        newMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
        newMethod.numBoxes = 10
        self.sampleResult.results.append(SubsamplingResult(newMethod))

        newMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
        newMethod.numBoxes = 15
        self.sampleResult.results.append(SubsamplingResult(newMethod))

        newMethod = gmeth.SpiralBoxSubsampling(None, 0.3)
        newMethod.numBoxes = 10
        self.sampleResult.results.append(SubsamplingResult(newMethod))
Josef Brandt's avatar
Josef Brandt committed
189 190 191

    def test_sampleResults_added_correctly(self):
        method: meth.SubsamplingMethod = self.sampleResult.results[0].method
192 193
        self.assertEqual(type(method), meth.RandomSampling)
        self.assertEqual(method.fraction, 0.1)
Josef Brandt's avatar
Josef Brandt committed
194 195

        method: meth.SubsamplingMethod = self.sampleResult.results[1].method
196 197 198
        self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
        self.assertEqual(method.fraction, 0.1)
        self.assertEqual(method.numBoxes, 10)
Josef Brandt's avatar
Josef Brandt committed
199 200

        method: meth.SubsamplingMethod = self.sampleResult.results[2].method
201 202 203 204 205 206 207 208
        self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
        self.assertEqual(method.fraction, 0.1)
        self.assertEqual(method.numBoxes, 15)

        method: meth.SubsamplingMethod = self.sampleResult.results[3].method
        self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
        self.assertEqual(method.fraction, 0.3)
        self.assertEqual(method.numBoxes, 10)
Josef Brandt's avatar
Josef Brandt committed
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224

    def test_result_is_already_present(self):
        newMethod: meth.SubsamplingMethod = meth.RandomSampling(None, 0.1)
        self.assertTrue(self.sampleResult._result_is_already_present(newMethod))
        newMethod: meth.SubsamplingMethod = meth.RandomSampling(None, 0.2)
        self.assertFalse(self.sampleResult._result_is_already_present(newMethod))

        newMethod: meth.SubsamplingMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
        self.assertTrue(self.sampleResult._result_is_already_present(newMethod))
        newMethod: meth.SubsamplingMethod = gmeth.SpiralBoxSubsampling(None, 0.2)
        self.assertFalse(self.sampleResult._result_is_already_present(newMethod))

        newMethod: meth.SubsamplingMethod = gmeth.CrossBoxSubSampling(None, 0.3)
        self.assertFalse(self.sampleResult._result_is_already_present(newMethod))

    def test_remove_result_of_method(self):
225
        numOrigResults = len(self.sampleResult.results)
Josef Brandt's avatar
Josef Brandt committed
226
        self.sampleResult._remove_result_of_method(meth.RandomSampling(None, 0.1))
227
        self.assertEqual(len(self.sampleResult.results), numOrigResults-1)
Josef Brandt's avatar
Josef Brandt committed
228 229

        self.sampleResult._remove_result_of_method(gmeth.SpiralBoxSubsampling(None, 0.1))
230
        self.assertEqual(len(self.sampleResult.results), numOrigResults-2)
Josef Brandt's avatar
Josef Brandt committed
231 232

        self.sampleResult._remove_result_of_method(gmeth.SpiralBoxSubsampling(None, 0.2))  # this is one is not present...
233 234 235 236 237
        self.assertEqual(len(self.sampleResult.results), numOrigResults-2)

    def test_attributes(self):
        self.sampleResult.set_attribute('soil')
        self.assertTrue(self.sampleResult.has_attribute('soil'))
238 239
        self.assertTrue(self.sampleResult.has_attribute('Soil'))  # we want to be case insensitive
        self.assertTrue(self.sampleResult.has_attribute('SOIL'))
240 241 242

        self.sampleResult.set_attribute('soil')  # the attribute is already there and shall not be added again
        self.assertEqual(len(self.sampleResult.attributes), 1)
Josef Brandt's avatar
Josef Brandt committed
243

244 245 246
        self.sampleResult.set_attribute('10µmFilter')
        self.assertEqual(len(self.sampleResult.attributes), 2)
        self.assertTrue(self.sampleResult.has_attribute('10µmFilter'))
Josef Brandt's avatar
Josef Brandt committed
247

248 249 250 251 252 253 254
        self.assertTrue(self.sampleResult.has_any_attribute(['soil', 'water']))
        self.assertTrue(self.sampleResult.has_any_attribute(['soil', 'water', '10µmFilter']))
        self.assertTrue(self.sampleResult.has_any_attribute(['water', '10µmFilter']))
        self.assertFalse(self.sampleResult.has_any_attribute(['water', 'sediment']))
        self.assertFalse(self.sampleResult.has_any_attribute(['beach']))


Josef Brandt's avatar
Josef Brandt committed
255
class TestSubsamplingResult(unittest.TestCase):
256
    def setUp(self):
Josef Brandt's avatar
Josef Brandt committed
257 258
        self.subsamplingResult: SubsamplingResult = SubsamplingResult(meth.RandomSampling(None, 0.1))

259
    def test_get_error_per_bin(self):
Josef Brandt's avatar
Josef Brandt committed
260 261
        def get_full_and_sub_particles():
            allParticles = []
262 263 264 265 266
            subParticles = []
            for particleSize in particleSizes:
                for _ in range(numParticlesPerSizeFull):
                    mpParticle = self._get_MP_particle()
                    mpParticle.longSize = mpParticle.shortSize = particleSize
Josef Brandt's avatar
Josef Brandt committed
267
                    allParticles.append(mpParticle)
268 269 270 271 272 273
                
                for _ in range(numParticlesPerSizeSub):
                    mpParticle = self._get_MP_particle()
                    mpParticle.longSize = mpParticle.shortSize = particleSize
                    subParticles.append(mpParticle)
            
Josef Brandt's avatar
Josef Brandt committed
274
            return allParticles, subParticles
275 276 277 278 279 280
                    
        binSizes = [5, 10, 20, 50, 100, 200, 500]
        particleSizes = [upperLimit - 1 for upperLimit in binSizes]
        
        numParticlesPerSizeFull = 20
        numParticlesPerSizeSub = 10
Josef Brandt's avatar
Josef Brandt committed
281
        fullParticles, subParticles = get_full_and_sub_particles()
282
        
Josef Brandt's avatar
Josef Brandt committed
283 284
        # assume everything was measured
        bins, mpCountErrorsPerBin = self.subsamplingResult._get_mp_count_error_per_bin(fullParticles, subParticles, 1.)
285 286 287 288

        for binIndex, binError in enumerate(mpCountErrorsPerBin):
            if binIndex <= 6:
                self.assertEqual(binError, 0.5)
Josef Brandt's avatar
Josef Brandt committed
289
            else:   # it's the last and largest bin, no particles where added there
290 291
                self.assertEqual(binError, 0)
        
Josef Brandt's avatar
Josef Brandt committed
292 293
        # assume only 50 % was measured
        bins, mpCountErrorsPerBin = self.subsamplingResult._get_mp_count_error_per_bin(fullParticles, subParticles, 0.5)
294 295 296 297 298 299 300 301 302 303 304 305

        for binIndex, binError in enumerate(mpCountErrorsPerBin):
            self.assertEqual(binError, 0)

    def test_get_number_of_MP_particles(self):
        mpParticles = self._get_MP_particles(5)
        numMPParticles = len(mpParticles)
        
        nonMPparticles = self._get_non_MP_particles(50)
        
        allParticles = mpParticles + nonMPparticles
        
Josef Brandt's avatar
Josef Brandt committed
306
        calculatedNumMPParticles = self.subsamplingResult._get_number_of_MP_particles(allParticles)
307 308 309 310 311 312 313 314 315
        self.assertEqual(numMPParticles, calculatedNumMPParticles)

    def test_get_mp_count_error(self):
        mpParticles1 = self._get_MP_particles(20)
        nonMPparticles1 = self._get_non_MP_particles(20)
        origParticles = mpParticles1 + nonMPparticles1
        
        mpParticles2 = self._get_MP_particles(30)
        estimateParticles = mpParticles2 + nonMPparticles1
Josef Brandt's avatar
Josef Brandt committed
316
        mpCountError = self.subsamplingResult._get_mp_count_error(origParticles, estimateParticles, 1.0)
317 318 319 320
        self.assertEqual(mpCountError, 0.5)
        
        mpParticles2 = self._get_MP_particles(20)
        estimateParticles = mpParticles2 + nonMPparticles1
Josef Brandt's avatar
Josef Brandt committed
321
        mpCountError = self.subsamplingResult._get_mp_count_error(origParticles, estimateParticles, 1.0)
322 323
        self.assertEqual(mpCountError, 0)
        
Josef Brandt's avatar
Josef Brandt committed
324
        mpCountError = self.subsamplingResult._get_mp_count_error(origParticles, estimateParticles, 0.5)
325 326 327 328
        self.assertEqual(mpCountError, 1.0)
    
    def test_get_error_from_values(self):
        exact, estimate = 100, 90
Josef Brandt's avatar
Josef Brandt committed
329
        error = self.subsamplingResult._get_error_from_values(exact, estimate)
330 331 332
        self.assertEqual(error, 0.1)
       
        exact, estimate = 100, 110
Josef Brandt's avatar
Josef Brandt committed
333
        error = self.subsamplingResult._get_error_from_values(exact, estimate)
334 335 336
        self.assertEqual(error, 0.1)
        
        exact, estimate = 100, 50
Josef Brandt's avatar
Josef Brandt committed
337
        error = self.subsamplingResult._get_error_from_values(exact, estimate)
338 339 340
        self.assertEqual(error, 0.5)
        
        exact, estimate = 100, 150
Josef Brandt's avatar
Josef Brandt committed
341
        error = self.subsamplingResult._get_error_from_values(exact, estimate)
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
        self.assertEqual(error, 0.5)        
        
    def _get_MP_particles(self, numParticles):
        mpParticles = []
        for _ in range(numParticles):
            mpParticles.append(self._get_MP_particle())
        return mpParticles
    
    def _get_non_MP_particles(self, numParticles):
        nonMPParticles = []
        for _ in range(numParticles):
            nonMPParticles.append(self._get_non_MP_particle())
        return nonMPParticles
    
    def _get_MP_particle(self):
        polymerNames = ['Poly (methyl methacrylate',
                        'Polyethylene',
                        'Silicone rubber',
                        'PB15',
                        'PY13',
                        'PR20']
        polymName = random.sample(polymerNames, 1)[0]
        newParticle = Particle()
        newMeas = Measurement()
        newMeas.setAssignment(polymName)
        newParticle.addMeasurement(newMeas)
        return newParticle

    def _get_non_MP_particle(self):
        newParticle = Particle()
        newParticle.addMeasurement(Measurement())        
        return newParticle