Commit 229dd4ac authored by Josef Brandt's avatar Josef Brandt

Basic concept of result objects

parent fa14c5fa
......@@ -6,30 +6,158 @@ Created on Wed Jan 22 13:57:28 2020
@author: luna
"""
import pickle
import sys
import os
from helpers import ParticleBinSorter
import methods as meth
import geometricMethods as gmeth
sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset
class ResultObject(object):
def get_name_from_directory(dirPath: str) -> str:
return str(os.path.basename(dirPath).split('.')[0])
class TotalResults(object):
methods: list = [meth.RandomSampling, meth.SizeBinFractioning, gmeth.CrossBoxSubSampling,
gmeth.SpiralBoxSubsampling]
measuredFreactions: list = [0.1, 0.3, 0.5, 0.9]
def __init__(self):
super(TotalResults, self).__init__()
self.sampleResults: list = []
def add_sample(self, filePath: str) -> None:
"""
Adds a new sampleResult object, if a .pkl file is given and if the sample name is not already present.
:param filePath:
:return:
"""
sampleName: str = get_name_from_directory(filePath)
presentSampleNames: list = [res.sampleName for res in self.sampleResults]
if sampleName not in presentSampleNames:
if os.path.basename(filePath).split('.')[-1] == 'pkl':
self.sampleResults.append(SampleResult(filePath))
def update_all(self) -> None:
"""
Updates all samples with all methods and all fractions
:return:
"""
for sample in self.sampleResults:
sample.load_dataset()
for fraction in self.measuredFreactions:
possibleMethods = self._get_methods_for_fraction(sample.dataset, fraction)
for curMethod in possibleMethods:
print(f'updating {sample.sampleName} with {curMethod.label} at fraction {fraction}')
sample.update_result_with_method(curMethod)
def _get_methods_for_fraction(self, dataset: dataset.DataSet, fraction: float) -> list:
"""
:param fraction: The desired fraction to measure
:return: list of measurement Objects that are applicable
"""
particleContainer = dataset.particleContainer
methods: list = [meth.RandomSampling(particleContainer, fraction),
meth.SizeBinFractioning(particleContainer, fraction)]
boxCreator: gmeth.BoxSelectionCreator = gmeth.BoxSelectionCreator(dataset)
methods += boxCreator.get_crossBoxSubsamplers_for_fraction(fraction)
methods += boxCreator.get_spiralBoxSubsamplers_for_fraction(fraction)
return methods
class SampleResult(object):
"""
An object the actually stores all generated results per sample and can update and report on them.
"""
def __init__(self, filepath: str):
super(ResultObject, self).__init__()
super(SampleResult, self).__init__()
self.filepath: str = filepath
print(self.sampleName)
self.dataset: dataset.DataSet = None
self.results: list = []
# noinspection PyTypeChecker
@property
def sampleName(self) -> str:
return os.path.basename(self.filepath).split('.')[0]
return get_name_from_directory(self.filepath)
def load_dataset(self) -> None:
self.dataset = dataset.loadData(self.filepath)
def update_result_with_method(self, method: meth.SubsamplingMethod, force: bool = False) -> None:
"""
Updates result with the given method (contains desiredFraction already)
:param method: The SubsamplingMethod Object
:param force: Wether to force an update. If False, the result is not updated, if it is already present.
:return:
"""
if not self._result_is_already_present(method) or force:
if force:
self._remove_result_of_method(method)
if self.dataset is None:
self.load_dataset()
method.particleContainer = self.dataset.particleContainer
newResult: SubsamplingResult = SubsamplingResult(method)
self.results.append(newResult)
newResult.update()
def _remove_result_of_method(self, method: meth.SubsamplingMethod) -> None:
"""
Removes the specified result from the list
:param method:
:return:
"""
for result in self.results:
if type(result.method) == type(method) and result.fraction == method.fraction:
self.results.remove(result)
def _result_is_already_present(self, method: meth.SubsamplingMethod) -> bool:
"""
Checks, if a result with the given method (method type AND measured fraction) is already present.
:param method: The method object, specifying the subsampling method and the measured fraction
:return:
"""
isPresent: bool = False
for result in self.results:
if type(result.method) == type(method) and result.fraction == method.fraction:
isPresent = True
break
return isPresent
class SubsamplingResult(object):
"""
Stores all interesting results from a subsampling experiment
"""
def __init__(self, subsamplingMethod: meth.SubsamplingMethod):
super(SubsamplingResult, self).__init__()
self.method: meth.SubsamplingMethod = subsamplingMethod
self.fraction = self.method.fraction
self.origParticleCount: int = None
self.subSampledParticleCount: int = None
self.mpCountError: float = None
self.mpCountErrorPerBin: tuple = None
def update(self) -> None:
"""
Updates all results from the method.
:return:
"""
assert self.method.particleContainer is not None
origParticles: list = self.method.particleContainer.particles
self.origParticleCount = len(origParticles)
subParticles: list = self.method.apply_subsampling_method()
self.subSampledParticleCount = len(subParticles)
fraction: float = self.method.fraction
self.mpCountError = self._get_mp_count_error(origParticles, subParticles, fraction)
self.mpCountErrorPerBin = self._get_mp_count_error_per_bin(origParticles, subParticles, fraction)
class ResultComparer(object):
def _get_mp_count_error_per_bin(self, allParticles, subParticles, fractionMeasured):
def _get_mp_count_error_per_bin(self, allParticles: list, subParticles: list, fractionMeasured: float) -> tuple:
binSorter = ParticleBinSorter()
allParticlesInBins = binSorter.sort_particles_into_bins(allParticles)
subParticlesInBins = binSorter.sort_particles_into_bins(subParticles)
......@@ -38,7 +166,7 @@ class ResultComparer(object):
mpCountErrorsPerBin.append(self._get_mp_count_error(allParticleBin, subParticleBin, fractionMeasured))
return binSorter.bins, mpCountErrorsPerBin
def _get_mp_count_error(self, allParticles, subParticles, fractionMeasured):
def _get_mp_count_error(self, allParticles: list, subParticles: list, fractionMeasured: float) -> float:
numMPOrig = self._get_number_of_MP_particles(allParticles)
numMPEstimate = self._get_number_of_MP_particles(subParticles) / fractionMeasured
......@@ -51,11 +179,11 @@ class ResultComparer(object):
return mpCountError
def _get_error_from_values(self, exact, estimate):
def _get_error_from_values(self, exact: float, estimate: float) -> float:
assert(exact != 0)
return abs(exact - estimate) / exact
def _get_number_of_MP_particles(self, particleList):
def _get_number_of_MP_particles(self, particleList: list) -> int:
mpPatterns = ['poly', 'rubber', 'pb', 'pr', 'pg', 'py', 'pv']
numMPParticles = 0
......
......@@ -55,62 +55,63 @@ class BoxSelectionSubsamplingMethod(SubsamplingMethod):
newTopLefts.append((topLeft[0] + self.offset[0], topLeft[1] + self.offset[1]))
return newTopLefts
class BoxSelectionCreator(object):
def __init__(self, dataset: dataset.DataSet):
super(BoxSelectionCreator, self).__init__()
self.dataset: dataset.DataSet = dataset
self.minNumberOfBoxes: int = 10
self.maxNumberOfBoxes: int = 20
def get_crossBoxSelectors_for_fraction(self, desiredFraction: float) -> list:
def get_crossBoxSubsamplers_for_fraction(self, desiredFraction: float) -> list:
"""
Creates CrossBoxSelectors that fullfill the desired fraction criterium.
Creates CrossBoxSubsamplers that fullfill the desired fraction criterium.
:param desiredFraction:
:return list of CrossBoxSelectors:
:return list of CrossBoxSubsamplers:
"""
crossBoxSelectors = []
crossBoxSubsamplers = []
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(self.dataset)
diameter = helpers.convert_length_to_pixels(self.dataset, diameter)
offset = helpers.convert_length_to_pixels(self.dataset, offset[0]), \
helpers.convert_length_to_pixels(self.dataset, offset[1])
for numBoxesAcross in [3, 5]:
newBoxSelector: CrossBoxSelector = CrossBoxSelector(self.dataset.particleContainer, desiredFraction)
newBoxSelector: CrossBoxSubSampling = CrossBoxSubSampling(self.dataset.particleContainer, desiredFraction)
newBoxSelector.filterDiameter = diameter
newBoxSelector.offset = offset
newBoxSelector.numBoxesAcross = numBoxesAcross
crossBoxSelectors.append(newBoxSelector)
maxFraction: float = newBoxSelector.get_maximum_achievable_fraction()
if desiredFraction <= maxFraction:
crossBoxSubsamplers.append(newBoxSelector)
return crossBoxSelectors
return crossBoxSubsamplers
def get_spiralBoxSelectors_for_fraction(self, desiredFraction: float) -> list:
def get_spiralBoxSubsamplers_for_fraction(self, desiredFraction: float) -> list:
"""
Creates CrossBoxSelectors that fullfill the desired fraction criterium.
Creates CrossBoxSubsamplers that fullfill the desired fraction criterium.
:param desiredFraction:
:return list of SpiralBoxSelectors:
"""
spiralBoxSelectors = []
spiralBoxSubsamplers = []
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(self.dataset)
diameter = helpers.convert_length_to_pixels(self.dataset, diameter)
offset = helpers.convert_length_to_pixels(self.dataset, offset[0]), \
helpers.convert_length_to_pixels(self.dataset, offset[1])
for numBoxes in SpiralSelector.possibleBoxNumbers:
newBoxSelector: SpiralSelector = SpiralSelector(self.dataset.particleContainer, desiredFraction)
for numBoxes in SpiralBoxSubsampling.possibleBoxNumbers:
newBoxSelector: SpiralBoxSubsampling = SpiralBoxSubsampling(self.dataset.particleContainer, desiredFraction)
newBoxSelector.filterDiameter = diameter
newBoxSelector.offset = offset
newBoxSelector.numBoxes = numBoxes
if newBoxSelector.noBoxOverlap:
spiralBoxSelectors.append(newBoxSelector)
spiralBoxSubsamplers.append(newBoxSelector)
return spiralBoxSelectors
return spiralBoxSubsamplers
class CrossBoxSelector(BoxSelectionSubsamplingMethod):
class CrossBoxSubSampling(BoxSelectionSubsamplingMethod):
def __init__(self, particleContainer, desiredFraction: float = 0.1) -> None:
super(CrossBoxSelector, self).__init__(particleContainer, desiredFraction)
super(CrossBoxSubSampling, self).__init__(particleContainer, desiredFraction)
self.numBoxesAcross: int = 3 # either 3 or 5
@property
......@@ -187,12 +188,12 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
return tileStarts
class SpiralSelector(BoxSelectionSubsamplingMethod):
possibleBoxNumbers: list = [10, 15, 20]
class SpiralBoxSubsampling(BoxSelectionSubsamplingMethod):
possibleBoxNumbers: list = [7, 10, 15]
def __init__(self, particleContainer, desiredFraction: float = 0.1) -> None:
super(SpiralSelector, self).__init__(particleContainer, desiredFraction)
self.numBoxes = 20
super(SpiralBoxSubsampling, self).__init__(particleContainer, desiredFraction)
self.numBoxes = 10
@property
def label(self) -> str:
......
from PyQt5 import QtCore, QtWidgets
from gui.filterView import FilterView, MeasureBoxGraphItem
from geometricMethods import BoxSelectionSubsamplingMethod, CrossBoxSelector, SpiralSelector
from geometricMethods import BoxSelectionSubsamplingMethod, CrossBoxSubSampling, SpiralBoxSubsampling
class MeasureMode(QtCore.QObject):
......@@ -26,7 +26,7 @@ class CrossBoxMode(MeasureMode):
def __init__(self, *args):
super(CrossBoxMode, self).__init__(*args)
self.uiControls = CrossBoxesControls(self)
self.boxGenerator: CrossBoxSelector = CrossBoxSelector(None)
self.boxGenerator: CrossBoxSubSampling = CrossBoxSubSampling(None)
self.update_measure_viewItems()
def update_measure_viewItems(self) -> None:
......@@ -91,7 +91,7 @@ class SpiralBoxMode(MeasureMode):
def __init__(self, *args):
super(SpiralBoxMode, self).__init__(*args)
self.uiControls: SpiralBoxControls = SpiralBoxControls(self)
self.boxGenerator: SpiralSelector = SpiralSelector(None)
self.boxGenerator: SpiralBoxSubsampling = SpiralBoxSubsampling(None)
self.update_measure_viewItems()
def update_measure_viewItems(self) -> None:
......
......@@ -32,6 +32,24 @@ class ParticleBinSorter(object):
return binIndex
def get_Anger_fraction(numParticles, sigma=1.65, mpFraction=0.01, errorMargin=0.1):
"""
Returns the required fraction for reaching the defined errorMargin at a given number of Particles.
According to: Anger et al. "Raman microspectroscopy as a tool for microplastic particle analysis",
TrAC Trends in Analytical Chemistry, 2018, 214-226. (https://doi.org/10.1016/j.trac.2018.10.010)
:param numParticles:
:param sigma:
:param mpFraction:
:param errorMargin:
:return:
"""
N = numParticles
P = mpFraction
e = P * errorMargin
numParticlesMeasured = np.ceil(P * (1 - P) / (e**2 / sigma**2 + P*(1 - P) / N))
return np.int(numParticlesMeasured)
def box_overlaps_contour(boxTopLeftXY: tuple, boxWidthHeight: tuple, contourData: np.ndarray) -> bool:
"""
Calculates, if a contour is overlapping a box.
......
......@@ -47,31 +47,6 @@ class RandomSampling(SubsamplingMethod):
def _get_number_of_random_particles(self, numTotalParticles):
return np.int(np.ceil(numTotalParticles * self.fraction))
class IvlevaSubsampling(SubsamplingMethod):
def __init__(self, particleContainer, sigma=1.65, mpFraction=0.01, errorMargin=0.1):
super(IvlevaSubsampling, self).__init__(particleContainer)
self.sigma = sigma
self.estimatedMPFraction = mpFraction
self.errorMargin = errorMargin
@property
def label(self) -> str:
return 'Random fraction, Anger et al.'
def apply_subsampling_method(self) -> list:
N = self.particleContainer.getNumberOfParticles()
numParticlesMeasured = self._get_ivleva_fraction(N)
subParticles = random.sample(self.particleContainer.particles, numParticlesMeasured)
fractionMeasured = numParticlesMeasured/N
return subParticles
def _get_ivleva_fraction(self, N):
P = self.estimatedMPFraction
e = P * self.errorMargin
numParticlesMeasured = np.ceil(P*(1 - P) / (e**2/self.sigma**2 + P*(1-P)/N))
return np.int(numParticlesMeasured)
class SizeBinFractioning(SubsamplingMethod):
......
......@@ -6,10 +6,10 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset
import gepardevaluation
from methods import IvlevaSubsampling, RandomSampling, SizeBinFractioning
from methods import RandomSampling, SizeBinFractioning
from geometricMethods import BoxSelectionCreator
from helpers import ParticleBinSorter
from evaluation import ResultComparer, ResultObject
from evaluation import TotalResults
"""
......@@ -25,70 +25,11 @@ workingFiles.append(r'C:\Users\xbrjos\Desktop\temp MP\190201_BSB_Stroomi_ds2_R1_
# These do not work, due to no ramanscansortindex??
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl'
for index, fname in enumerate(workingFiles):
# dset = dataset.loadData(fname)
newObj: ResultObject = ResultObject(fname)
# print('loaded dataset', fname)
# boxCreator = BoxSelectionCreator(dset)
# center, size = boxCreator.get_filterDimensions_from_dataset()
# print(center, size)
# print(dset.mapToPixel(center, force=True))
# pc = dset.particleContainer
# origParticles = pc.particles
# resultComparer = ResultComparer()
# numOrigMP = resultComparer._get_number_of_MP_particles(origParticles)
# print(f'orig particles: {len(origParticles)}, of which are mp: {numOrigMP}')
# # ivlevaSampling = IvlevaSubsampling(pc)
# # ivlevaFraction, ivlevaParticles = ivlevaSampling.apply_subsampling_method()
# t0 = time.time()
# fractions = np.arange(0.05, .55, 0.05)
# errors = []
# binErrors = []
# numIterations = 1000
results: TotalResults = TotalResults()
# for fraction in fractions:
# print('random sampling, fraction:', fraction)
# # randomSampling = RandomSampling(pc, desiredFraction=fraction)
# randomSampling = SizeBinFractioning(pc, fraction)
# iterErrors = []
# binIterErrors = []
# for _ in range(numIterations):
# randomFraction, randomParticles = randomSampling.apply_subsampling_method()
# iterErrors.append(resultComparer._get_mp_count_error(origParticles, randomParticles, randomFraction))
# bins, errorsPerBin = resultComparer._get_mp_count_error_per_bin(origParticles, randomParticles, randomFraction)
# binIterErrors.append(errorsPerBin)
# errors.append(round(np.mean(iterErrors)*100)) #from fraction to %
# fractionBinErrors = []
# for binIndex in range(len(bins)+1):
# binError = round(np.mean([binIterErrors[i][binIndex] for i in range(numIterations)]) * 100)
# fractionBinErrors.append(binError)
# binErrors.append(fractionBinErrors)
# print('random sampling took', np.round(time.time()-t0, 2), 'seonds')
# binLowerLimits = bins.copy()
# binLowerLimits.insert(0, 0)
# plt.subplot(121)
# plt.plot(fractions, errors)
# # plt.title(f'Random Sampling, averaged from {numIterations} trials, orig particle count: {len(origParticles)}')
# plt.xlabel('Fraction measured')
# plt.ylabel('Average error in MP particle count (%)')
# plt.subplot(122)
# for fracMeas, curBinErrors in zip(fractions, binErrors):
# plt.plot(binLowerLimits, curBinErrors, label=np.round(fracMeas, 1))
# # plt.title('Error in MP count (%) per size bin')
# plt.xlabel('particle size')
# plt.ylabel('Average error in MP particle count (%)')
# plt.legend()
# plt.show()
# # sizeBinSampling = SizeBinFractioning(pc)
# # sizeBinParticles = sizeBinSampling.apply_subsampling_method()
\ No newline at end of file
for index, fname in enumerate(workingFiles):
results.add_sample(fname)
t0 = time.time()
results.update_all()
print('updating all took', time.time()-t0, 'seconds')
This diff is collapsed.
This diff is collapsed.
......@@ -68,6 +68,25 @@ class TestBinSorter(unittest.TestCase):
class TestOther(unittest.TestCase):
def test_get_Anger_fraction(self):
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.05, errorMargin=0.1)
self.assertEqual(numParticles, 5147)
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.005, errorMargin=0.1)
self.assertEqual(numParticles, 51394)
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.0005, errorMargin=0.1)
self.assertEqual(numParticles, 352428)
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.05, errorMargin=0.2)
self.assertEqual(numParticles, 1292)
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.005, errorMargin=0.3)
self.assertEqual(numParticles, 5984)
numParticles = helpers.get_Anger_fraction(1E6, sigma=1.65, mpFraction=0.0005, errorMargin=0.3)
self.assertEqual(numParticles, 57022)
def test_box_overlaps_contour(self):
boxXY: tuple = 0, 0
boxWidthHeight: tuple = 10, 10
......
......@@ -13,7 +13,7 @@ import numpy as np
import gepard
from gepard.analysis.particleContainer import ParticleContainer
from gepard.analysis.particleAndMeasurement import Particle
from methods import IvlevaSubsampling, RandomSampling, SizeBinFractioning
from methods import RandomSampling, SizeBinFractioning
from helpers import ParticleBinSorter
......@@ -23,28 +23,6 @@ def get_default_particle_container(numParticles=1000):
return particleContainer
class TestIvleva(unittest.TestCase):
def test_get_ivleva_fraction(self):
self.particleContainer = get_default_particle_container()
ivlevaSampling = IvlevaSubsampling(self.particleContainer, sigma=1.65, mpFraction=0.05, errorMargin=0.1)
numParticles = ivlevaSampling._get_ivleva_fraction(1E6)
self.assertEqual(numParticles, 5147)
ivlevaSampling = IvlevaSubsampling(self.particleContainer, sigma=1.65, mpFraction=0.005, errorMargin=0.1)
numParticles = ivlevaSampling._get_ivleva_fraction(1E6)
self.assertEqual(numParticles, 51394)
ivlevaSampling = IvlevaSubsampling(self.particleContainer, sigma=1.65, mpFraction=0.0005, errorMargin=0.1)
numParticles = ivlevaSampling._get_ivleva_fraction(1E6)
self.assertEqual(numParticles, 352428)
ivlevaSampling = IvlevaSubsampling(self.particleContainer, sigma=1.65, mpFraction=0.05, errorMargin=0.1)
numParticles = ivlevaSampling._get_ivleva_fraction(1000)
ivlevaParticles = ivlevaSampling.apply_subsampling_method()
self.assertEqual(len(ivlevaParticles), numParticles)
class TestRandomParticles(unittest.TestCase):
def test_get_number_of_random_particles(self):
randomSampling = RandomSampling(None, desiredFraction=0.1)
......@@ -85,7 +63,3 @@ class TestSizeBinFractioning(unittest.TestCase):
subParticlesPerBin: list = self.sizeBinFrac._get_subParticles_per_bin(self.particles)
for subParticles in subParticlesPerBin:
self.assertEqual(len(subParticles), numParticlesPerBinExpected)
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment