...
 
Commits (9)
......@@ -4,3 +4,5 @@
__pycache__/
*.png
*.res
......@@ -5,12 +5,226 @@ Created on Wed Jan 22 13:57:28 2020
@author: luna
"""
import pickle
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
from helpers import ParticleBinSorter
import methods as meth
import geometricMethods as gmeth
sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset
def get_name_from_directory(dirPath: str) -> str:
return str(os.path.basename(dirPath).split('.')[0])
class TotalResults(object):
methods: list = [meth.RandomSampling, meth.SizeBinFractioning, gmeth.CrossBoxSubSampling,
gmeth.SpiralBoxSubsampling]
measuredFreactions: list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 0.9]
def __init__(self):
super(TotalResults, self).__init__()
self.sampleResults: list = []
def add_sample(self, filePath: str):
"""
Adds a new sampleResult object, if a .pkl file is given and if the sample name is not already present.
:param filePath:
:return:
"""
newResult: SampleResult = None
sampleName: str = get_name_from_directory(filePath)
presentSampleNames: list = [res.sampleName for res in self.sampleResults]
if sampleName not in presentSampleNames:
if os.path.basename(filePath).split('.')[-1] == 'pkl':
newResult = SampleResult(filePath)
self.sampleResults.append(newResult)
return newResult
def update_all(self) -> None:
"""
Updates all samples with all methods and all fractions
:return:
"""
for index, sample in enumerate(self.sampleResults):
sample.load_dataset()
for fraction in self.measuredFreactions:
possibleMethods = self._get_methods_for_fraction(sample.dataset, fraction)
for curMethod in possibleMethods:
# print(f'updating {sample.sampleName} with {curMethod.label} at fraction {fraction}')
sample.update_result_with_method(curMethod)
print(f'processed {index+1} of {len(self.sampleResults)} samples')
def get_error_vs_fraction_data(self, attributes: list = [], methods: list = []) -> dict:
"""
Returns Dict: Key: Method Label, Value: (Dict: Key:Measured Fraction, Value: averaged MPCountError over all samples)
:param attributes: A list of attributes that should be used for filtering the samples. Only samples with an
attribute from within that list are considered.
:return:
"""
result: dict = {}
for sample in self.sampleResults:
sample: SampleResult = sample
if attributes == [] or sample.has_any_attribute(attributes):
for res in sample.results:
res: SubsamplingResult = res
method: meth.SubsamplingMethod = res.method
if methods == [] or method.matches_any_pattern(methods):
label: str = method.label
frac: float = method.fraction
error: float = res.mpCountError
if label not in result.keys():
result[label] = {frac: [error]}
elif frac not in result[label].keys():
result[label][frac] = [error]
else:
result[label][frac].append(error)
for method in result.keys():
methodRes: dict = result[method]
for fraction in methodRes.keys():
methodRes[fraction] = np.mean(methodRes[fraction])
return result
def _get_methods_for_fraction(self, dataset: dataset.DataSet, fraction: float) -> list:
"""
:param fraction: The desired fraction to measure
:return: list of measurement Objects that are applicable
"""
particleContainer = dataset.particleContainer
methods: list = [meth.RandomSampling(particleContainer, fraction),
meth.SizeBinFractioning(particleContainer, fraction)]
boxCreator: gmeth.BoxSelectionCreator = gmeth.BoxSelectionCreator(dataset)
methods += boxCreator.get_crossBoxSubsamplers_for_fraction(fraction)
methods += boxCreator.get_spiralBoxSubsamplers_for_fraction(fraction)
return methods
class SampleResult(object):
"""
An object the actually stores all generated results per sample and can update and report on them.
"""
def __init__(self, filepath: str):
super(SampleResult, self).__init__()
self.filepath: str = filepath
self.dataset: dataset.DataSet = None
self.results: list = []
self.attributes: list = []
@property
def sampleName(self) -> str:
return get_name_from_directory(self.filepath)
def load_dataset(self) -> None:
self.dataset = dataset.loadData(self.filepath)
assert self.dataset is not None
def update_result_with_method(self, method: meth.SubsamplingMethod, force: bool = False) -> None:
"""
Updates result with the given method (contains desiredFraction already)
:param method: The SubsamplingMethod Object
:param force: Wether to force an update. If False, the result is not updated, if it is already present.
:return:
"""
if not self._result_is_already_present(method) or force:
if force:
self._remove_result_of_method(method)
if self.dataset is None:
self.load_dataset()
method.particleContainer = self.dataset.particleContainer
newResult: SubsamplingResult = SubsamplingResult(method)
self.results.append(newResult)
newResult.update()
def set_attribute(self, newAttribute: str) -> None:
"""
Adds a new attribute to the sample, if it does not contain the attribute already
:param newAttribute:
:return:
"""
if not self.has_attribute(newAttribute):
self.attributes.append(newAttribute)
print(f'sample {self.filepath} has now attribute {newAttribute}')
def has_any_attribute(self, listOfAttributes: list) -> bool:
hasAttr: bool = False
for attr in listOfAttributes:
if self.has_attribute(attr):
hasAttr = True
break
return hasAttr
def has_attribute(self, attribute: str) -> bool:
attributes: list = [attr.lower() for attr in self.attributes]
return attribute.lower() in attributes
def _remove_result_of_method(self, method: meth.SubsamplingMethod) -> None:
"""
Removes the specified result from the list
:param method:
:return:
"""
for result in self.results:
if method.equals(result.method):
self.results.remove(result)
def _result_is_already_present(self, method: meth.SubsamplingMethod) -> bool:
"""
Checks, if a result with the given method (method type AND measured fraction) is already present.
:param method: The method object, specifying the subsampling method and the measured fraction
:return:
"""
isPresent: bool = False
for result in self.results:
if method.equals(result.method):
isPresent = True
break
return isPresent
class SubsamplingResult(object):
"""
Stores all interesting results from a subsampling experiment
"""
def __init__(self, subsamplingMethod: meth.SubsamplingMethod):
super(SubsamplingResult, self).__init__()
self.method: meth.SubsamplingMethod = subsamplingMethod
self.fraction = self.method.fraction
self.origParticleCount: int = None
self.subSampledParticleCount: int = None
self.mpCountError: float = None
self.mpCountErrorPerBin: tuple = None
def update(self) -> None:
"""
Updates all results from the method.
:return:
"""
assert self.method.particleContainer is not None
origParticles: list = self.method.particleContainer.particles
self.origParticleCount = len(origParticles)
subParticles: list = self.method.apply_subsampling_method()
self.subSampledParticleCount = len(subParticles)
fraction: float = self.method.fraction
class ResultComparer(object):
self.mpCountError = self._get_mp_count_error(origParticles, subParticles, fraction)
# print(f'{self.origParticleCount} particles, thereof {self.subSampledParticleCount} measured, error: {self.mpCountError}')
self.mpCountErrorPerBin = self._get_mp_count_error_per_bin(origParticles, subParticles, fraction)
# print(f'method {self.method.label} updated, result is {self.mpCountError}')
def _get_mp_count_error_per_bin(self, allParticles, subParticles, fractionMeasured):
def _get_mp_count_error_per_bin(self, allParticles: list, subParticles: list, fractionMeasured: float) -> tuple:
binSorter = ParticleBinSorter()
allParticlesInBins = binSorter.sort_particles_into_bins(allParticles)
subParticlesInBins = binSorter.sort_particles_into_bins(subParticles)
......@@ -19,7 +233,7 @@ class ResultComparer(object):
mpCountErrorsPerBin.append(self._get_mp_count_error(allParticleBin, subParticleBin, fractionMeasured))
return binSorter.bins, mpCountErrorsPerBin
def _get_mp_count_error(self, allParticles, subParticles, fractionMeasured):
def _get_mp_count_error(self, allParticles: list, subParticles: list, fractionMeasured: float) -> float:
numMPOrig = self._get_number_of_MP_particles(allParticles)
numMPEstimate = self._get_number_of_MP_particles(subParticles) / fractionMeasured
......@@ -32,11 +246,11 @@ class ResultComparer(object):
return mpCountError
def _get_error_from_values(self, exact, estimate):
def _get_error_from_values(self, exact: float, estimate: float) -> float:
assert(exact != 0)
return abs(exact - estimate) / exact
def _get_number_of_MP_particles(self, particleList):
def _get_number_of_MP_particles(self, particleList: list) -> int:
mpPatterns = ['poly', 'rubber', 'pb', 'pr', 'pg', 'py', 'pv']
numMPParticles = 0
......@@ -47,4 +261,4 @@ class ResultComparer(object):
numMPParticles += 1
break
return numMPParticles
\ No newline at end of file
return numMPParticles
......@@ -2,8 +2,10 @@ import numpy as np
from itertools import combinations
from methods import SubsamplingMethod
import sys
sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset
import helpers
class BoxSelectionSubsamplingMethod(SubsamplingMethod):
......@@ -12,55 +14,112 @@ class BoxSelectionSubsamplingMethod(SubsamplingMethod):
self.filterDiameter: float = 500
self.offset: tuple = (0, 0)
@property
def label(self) -> str:
raise NotImplementedError
@property
def filterArea(self) -> float:
return np.pi * (self.filterDiameter / 2) ** 2
def apply_subsampling_method(self) -> tuple:
def apply_subsampling_method(self) -> list:
def distanceToCnt(topleft: tuple):
return abs(topleft[0] - cntStart[0]) + abs(topleft[1] - cntStart[1])
subParticles: list = []
topLefts: list = self.get_topLeft_of_boxes()
boxSize = self.boxSize
boxWidthHeight: tuple = (self.boxSize, self.boxSize)
for particle in self.particleContainer.particles:
for topLeft in topLefts:
if box_overlaps_contour(topLeft, (boxSize, boxSize), particle.contour, self.offset):
cntStart: tuple = (particle.contour[0, 0, 0], particle.contour[0, 0, 1])
sortedTopLefts = sorted(topLefts, key=distanceToCnt)
for topLeftXY in sortedTopLefts:
if helpers.box_overlaps_contour(topLeftXY, boxWidthHeight, particle.contour):
subParticles.append(particle)
break
return self.fraction, subParticles
return subParticles
def get_topLeft_of_boxes(self) -> list:
raise NotImplementedError
def _apply_offset_to_toplefts(self, topLefts: list) -> list:
"""
Applies the filter offset to the calculated topLefts of the measure boxes.
:param topLefts:
:return:
"""
newTopLefts: list = []
for topLeft in topLefts:
newTopLefts.append((topLeft[0] + self.offset[0], topLeft[1] + self.offset[1]))
return newTopLefts
def equals(self, otherMethod) -> bool:
raise NotImplementedError
class BoxSelectionCreator(object):
def __init__(self, dataset: dataset.DataSet):
super(BoxSelectionCreator, self).__init__()
self.dataset: dataset.DataSet = dataset
def get_crossBoxSelectors_for_fraction(self, desiredFraction: float) -> list:
def get_crossBoxSubsamplers_for_fraction(self, desiredFraction: float) -> list:
"""
Creates CrossBoxSelectors that fullfil the desired fraction criterium.
Creates CrossBoxSubsamplers that fullfill the desired fraction criterium.
:param desiredFraction:
:return:
:return list of CrossBoxSubsamplers:
"""
crossBoxSelectors = []
offset, diameter, widthHeight = self.get_filterDimensions_from_dataset()
crossBoxSubsamplers = []
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(self.dataset)
diameter = helpers.convert_length_to_pixels(self.dataset, diameter)
offset = helpers.convert_length_to_pixels(self.dataset, offset[0]), \
helpers.convert_length_to_pixels(self.dataset, offset[1])
for numBoxesAcross in [3, 5]:
newBoxSelector: CrossBoxSubSampling = CrossBoxSubSampling(self.dataset.particleContainer, desiredFraction)
newBoxSelector.filterDiameter = diameter
newBoxSelector.offset = offset
newBoxSelector.numBoxesAcross = numBoxesAcross
# for numBoxesAcross in [3, 5]:
# newBoxSelector: CrossBoxSelector = CrossBoxSelector(self.dataset.particleContainer, desiredFraction)
# newBoxSelector.filterDiameter = diameter
# newBoxSelector.offset = offset
# newBoxSelector.numBoxesAcross = numBoxesAcross
#
# crossBoxSelectors.append(newBoxSelector)
maxFraction: float = newBoxSelector.get_maximum_achievable_fraction()
if desiredFraction <= maxFraction:
crossBoxSubsamplers.append(newBoxSelector)
return crossBoxSelectors
return crossBoxSubsamplers
def get_spiralBoxSubsamplers_for_fraction(self, desiredFraction: float) -> list:
"""
Creates CrossBoxSubsamplers that fullfill the desired fraction criterium.
:param desiredFraction:
:return list of SpiralBoxSelectors:
"""
spiralBoxSubsamplers = []
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(self.dataset)
diameter = helpers.convert_length_to_pixels(self.dataset, diameter)
offset = helpers.convert_length_to_pixels(self.dataset, offset[0]), \
helpers.convert_length_to_pixels(self.dataset, offset[1])
for numBoxes in SpiralBoxSubsampling.possibleBoxNumbers:
newBoxSelector: SpiralBoxSubsampling = SpiralBoxSubsampling(self.dataset.particleContainer, desiredFraction)
newBoxSelector.filterDiameter = diameter
newBoxSelector.offset = offset
newBoxSelector.numBoxes = numBoxes
class CrossBoxSelector(BoxSelectionSubsamplingMethod):
if newBoxSelector.noBoxOverlap:
spiralBoxSubsamplers.append(newBoxSelector)
return spiralBoxSubsamplers
class CrossBoxSubSampling(BoxSelectionSubsamplingMethod):
def __init__(self, particleContainer, desiredFraction: float = 0.1) -> None:
super(CrossBoxSelector, self).__init__(particleContainer, desiredFraction)
self.numBoxesAcross: int = 3 # either 3 or 5
super(CrossBoxSubSampling, self).__init__(particleContainer, desiredFraction)
self.numBoxesAcross: int = 3 # either 3 or 5
@property
def label(self) -> str:
return f'Boxes CrossLayout ({self.numBoxesAcross} boxes across)'
@property
def boxSize(self) -> float:
......@@ -69,23 +128,23 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
self.fraction = maxFraction
totalBoxArea: float = self.filterArea * self.fraction
boxArea: float = totalBoxArea / (2*self.numBoxesAcross - 1)
return boxArea**0.5
boxArea: float = totalBoxArea / (2 * self.numBoxesAcross - 1)
return boxArea ** 0.5
def get_topLeft_of_boxes(self) -> list:
topLeftCorners: list = []
boxSize = self.boxSize
xStartCoordinates: list = self._get_horizontal_box_starts(boxSize)
yStartCoordinates: list = self._get_vertical_box_starts(boxSize)
middleXCoordinate: float = xStartCoordinates[self.numBoxesAcross//2]
middleYCoordinate: float = yStartCoordinates[self.numBoxesAcross//2]
middleXCoordinate: float = xStartCoordinates[self.numBoxesAcross // 2]
middleYCoordinate: float = yStartCoordinates[self.numBoxesAcross // 2]
for i in range(self.numBoxesAcross):
topLeftCorners.append((middleXCoordinate, yStartCoordinates[i]))
if i != self.numBoxesAcross//2:
if i != self.numBoxesAcross // 2:
topLeftCorners.append((xStartCoordinates[i], middleYCoordinate))
return topLeftCorners
return self._apply_offset_to_toplefts(topLeftCorners)
def get_maximum_achievable_fraction(self) -> float:
"""
......@@ -94,17 +153,24 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
:return float:
"""
alpha: float = np.deg2rad(135)
r: float = self.filterDiameter/2
d: float = (self.numBoxesAcross-1) * r / self.numBoxesAcross # 2/3*r for numAcross = 3, 4/5*r numAcross = 5
r: float = self.filterDiameter / 2
d: float = (self.numBoxesAcross - 1) * r / self.numBoxesAcross # 2/3*r for numAcross = 3, 4/5*r numAcross = 5
delta: float = np.arcsin((np.sin(alpha) * d) / r)
gamma: float = np.pi - alpha - delta
longestBoxHalfDiagonal: float = r / np.sin(alpha) * np.sin(gamma)
maxBoxSize: float = 2 * longestBoxHalfDiagonal / np.sqrt(2)
numBoxes: int = 2*self.numBoxesAcross - 1
totalBoxArea: float = numBoxes * (maxBoxSize**2)
numBoxes: int = 2 * self.numBoxesAcross - 1
totalBoxArea: float = numBoxes * (maxBoxSize ** 2)
maxFraction: float = totalBoxArea / self.filterArea
return maxFraction
def equals(self, otherMethod) -> bool:
equals: bool = False
if type(otherMethod) == type(self) and otherMethod.fraction == self.fraction:
if otherMethod.numBoxesAcross == self.numBoxesAcross:
equals = True
return equals
def _get_horizontal_box_starts(self, boxSize: float) -> list:
"""
Returns a list of width-values at which the individual boxes start
......@@ -132,30 +198,41 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
return tileStarts
class SpiralSelector(BoxSelectionSubsamplingMethod):
class SpiralBoxSubsampling(BoxSelectionSubsamplingMethod):
possibleBoxNumbers: list = [7, 10, 15]
def __init__(self, particleContainer, desiredFraction: float = 0.1) -> None:
super(SpiralSelector, self).__init__(particleContainer, desiredFraction)
self.boxSize: float = 50
self.numBoxes = 20
super(SpiralBoxSubsampling, self).__init__(particleContainer, desiredFraction)
self.numBoxes = 10
@property
def label(self) -> str:
return f'Boxes SpiralLayout ({self.numBoxes} boxes)'
@property
def noBoxOverlap(self) -> bool:
return not self._boxes_are_overlapping(self.get_topLeft_of_boxes())
@property
def boxSize(self) -> float:
totalBoxArea: float = self.filterArea * self.fraction
boxArea: float = totalBoxArea / self.numBoxes
return boxArea ** 0.5
@property
def spiralSlope(self) -> float:
return self.armDistance / (2*np.pi)
return self.armDistance / (2 * np.pi)
@property
def armDistance(self) -> float:
return np.sqrt(2) * self.boxSize
@property
def actuallyCoveredFraction(self) -> float:
return self.numBoxes*self.boxSize**2 / self.filterArea
def get_topLeft_of_boxes(self) -> list:
"""
Calculates the topLeft-points (x, y) of all measure boxes
The method uses an approximation for the spiral and is not purely accurate.
:return list:"""
filterCenter: tuple = self.filterDiameter/2 - self.boxSize/2, self.filterDiameter/2 - self.boxSize/2
filterCenter: tuple = self.filterDiameter / 2 - self.boxSize / 2, self.filterDiameter / 2 - self.boxSize / 2
slope = self.spiralSlope
theta: float = 0
boxDistance = self.boxSize * 1.1
......@@ -164,11 +241,18 @@ class SpiralSelector(BoxSelectionSubsamplingMethod):
for i in range(self.numBoxes):
newPoint: tuple = self._get_xy_at_angle(theta, filterCenter)
topLefts.append(newPoint)
theta += boxDistance / (slope * np.sqrt(1 + theta**2))
theta += boxDistance / (slope * np.sqrt(1 + theta ** 2))
boxDistance *= 1.05
topLefts = self._move_and_scale_toplefts(topLefts)
return topLefts
return self._apply_offset_to_toplefts(topLefts)
def equals(self, otherMethod) -> bool:
equals: bool = False
if type(otherMethod) == type(self) and otherMethod.fraction == self.fraction:
if otherMethod.numBoxes == self.numBoxes:
equals = True
return equals
def _move_and_scale_toplefts(self, topLefts: list) -> list:
"""
......@@ -176,11 +260,11 @@ class SpiralSelector(BoxSelectionSubsamplingMethod):
This function moves and scales the topLeft-Points so that all measure boxes lie within the filter limits.
:return list:
"""
xCoords: np.ndarray= np.array([float(point[0]) for point in topLefts]) - self.filterDiameter / 2
yCoords: np.ndarray= np.array([float(point[1]) for point in topLefts]) - self.filterDiameter / 2
xCoords: np.ndarray = np.array([float(point[0]) for point in topLefts]) - self.filterDiameter / 2
yCoords: np.ndarray = np.array([float(point[1]) for point in topLefts]) - self.filterDiameter / 2
xCoordsBoxMiddles: np.ndarray= xCoords + self.boxSize/2
yCoordsBoxMiddles: np.ndarray= yCoords + self.boxSize/2
xCoordsBoxMiddles: np.ndarray = xCoords + self.boxSize / 2
yCoordsBoxMiddles: np.ndarray = yCoords + self.boxSize / 2
lastBoxCenter: tuple = (xCoordsBoxMiddles[-1], yCoordsBoxMiddles[-1])
distanceLastCenter: float = np.linalg.norm(lastBoxCenter)
......@@ -193,8 +277,8 @@ class SpiralSelector(BoxSelectionSubsamplingMethod):
xCoordsBoxMiddles *= scaleFactor
yCoordsBoxMiddles *= scaleFactor
xCoords = xCoordsBoxMiddles + (self.filterDiameter - self.boxSize)/2
yCoords = yCoordsBoxMiddles + (self.filterDiameter - self.boxSize)/2
xCoords = xCoordsBoxMiddles + (self.filterDiameter - self.boxSize) / 2
yCoords = yCoordsBoxMiddles + (self.filterDiameter - self.boxSize) / 2
newTopLefts = zip(np.round(xCoords), np.round(yCoords))
return list(tuple(newTopLefts))
......@@ -208,12 +292,12 @@ class SpiralSelector(BoxSelectionSubsamplingMethod):
"""
center = np.array(center)
boxSize = self.boxSize
coords: np.ndarray= np.array([[boxCenter[0] - 0.5*boxSize, boxCenter[1] - 0.5*boxSize],
[boxCenter[0] + 0.5*boxSize, boxCenter[1] - 0.5*boxSize],
[boxCenter[0] - 0.5*boxSize, boxCenter[1] + 0.5*boxSize],
[boxCenter[0] + 0.5*boxSize, boxCenter[1] + 0.5*boxSize]])
coords: np.ndarray = np.array([[boxCenter[0] - 0.5 * boxSize, boxCenter[1] - 0.5 * boxSize],
[boxCenter[0] + 0.5 * boxSize, boxCenter[1] - 0.5 * boxSize],
[boxCenter[0] - 0.5 * boxSize, boxCenter[1] + 0.5 * boxSize],
[boxCenter[0] + 0.5 * boxSize, boxCenter[1] + 0.5 * boxSize]])
distances: np.ndarray= np.linalg.norm(coords - center, axis=1)
distances: np.ndarray = np.linalg.norm(coords - center, axis=1)
return np.max(distances)
def _get_xy_at_angle(self, theta: float, centerXY: tuple = (0, 0)) -> tuple:
......
......@@ -4,6 +4,7 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
import gepard
from gepard import dataset
import helpers
import numpy as np
class FilterView(QtWidgets.QGraphicsView):
......@@ -21,8 +22,6 @@ class FilterView(QtWidgets.QGraphicsView):
self.setCacheMode(QtWidgets.QGraphicsView.CacheBackground)
self.setViewportUpdateMode(QtWidgets.QGraphicsView.BoundingRectViewportUpdate)
self.setRenderHint(QtGui.QPainter.Antialiasing)
self.setTransformationAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
self.setResizeAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
self.drag = None
......@@ -33,28 +32,18 @@ class FilterView(QtWidgets.QGraphicsView):
def update_measure_boxes(self, topLefts: list, boxSize: float) -> None:
self._remove_measure_boxes()
offset = self.filter.circleOffset
for x, y in topLefts:
newBox: MeasureBoxGraphItem = MeasureBoxGraphItem(x+offset[0], y+offset[1], boxSize, boxSize)
newBox: MeasureBoxGraphItem = MeasureBoxGraphItem(x, y, boxSize, boxSize)
self.measuringBoxes.append(newBox)
self.scene().addItem(newBox)
self._update_measured_contours()
def _remove_measure_boxes(self) -> None:
for item in self.measuringBoxes:
self.scene().removeItem(item)
self.measuringBoxes = []
def load_and_update_from_dataset(self, fname: str) -> None:
self.dataset = dataset.loadData(fname)
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(self.dataset)
offsetx = helpers.convert_length_to_pixels(self.dataset, offset[0])
offsety = helpers.convert_length_to_pixels(self.dataset, offset[1])
diameter = helpers.convert_length_to_pixels(self.dataset, diameter)
width = helpers.convert_length_to_pixels(self.dataset, widthHeight[0])
height = helpers.convert_length_to_pixels(self.dataset, widthHeight[1])
self.filter.update_filterSize(width, height, diameter, (offsetx, offsety))
def update_from_dataset(self, dset: dataset.DataSet) -> None:
self.dataset = dset
self._update_particle_contours()
self._fit_to_window()
......@@ -71,21 +60,10 @@ class FilterView(QtWidgets.QGraphicsView):
self.scene().removeItem(cntItem)
self.contourItems = []
def _update_measured_contours(self) -> None:
# offset = self.filter.circleOffset
# offset = (-offset[0], -offset[1])
offset = (0, 0)
for contourItem in self.contourItems:
contourItem.isMeasured = False
for measBox in self.measuringBoxes:
topLeftXY = (measBox.posX, measBox.posY)
boxWidthHeight = (measBox.width, measBox.height)
if helpers.box_overlaps_contour(topLeftXY, boxWidthHeight, contourItem.polygon, offset=offset):
contourItem.isMeasured = True
contourItem.update()
break
def update_measured_particles(self, measuredParticles: list) -> None:
measuredIndices: list = [particle.index for particle in measuredParticles]
for index, contourItem in enumerate(self.contourItems):
contourItem.isMeasured = (index in measuredIndices)
def wheelEvent(self, event: QtGui.QWheelEvent) -> None:
factor: float = 1.01 ** (event.angleDelta().y() / 8)
......
from PyQt5 import QtWidgets
import sys
sys.path.append("C://Users//xbrjos//Desktop//Python")
import gepard
from gepard import dataset
from gui.filterView import FilterView
from gui.measureModes import MeasureMode, CrossBoxMode, CrossBoxesControls, SpiralBoxMode
import helpers
class MainView(QtWidgets.QWidget):
......@@ -64,8 +69,27 @@ class MainView(QtWidgets.QWidget):
def _load_dataset(self) -> None:
fname = QtWidgets.QFileDialog.getOpenFileName(self, 'Select .pkl file', filter='pkl file (*.pkl)')
if fname[0] != '':
self.filterView.load_and_update_from_dataset(fname[0])
self.activeMode.update_measure_viewItems()
try:
dset: dataset.DataSet = dataset.loadData(fname[0])
except IndexError:
dset = None
QtWidgets.QMessageBox.critical(self, 'Index Error', 'Unable to load dataset..')
if dset is not None:
offset, diameter, widthHeight = helpers.get_filterDimensions_from_dataset(dset)
offsetx = helpers.convert_length_to_pixels(dset, offset[0])
offsety = helpers.convert_length_to_pixels(dset, offset[1])
diameter = helpers.convert_length_to_pixels(dset, diameter)
width = helpers.convert_length_to_pixels(dset, widthHeight[0])
height = helpers.convert_length_to_pixels(dset, widthHeight[1])
self.filterView.filter.update_filterSize(width, height, diameter, (offsetx, offsety))
for mode in self.measureModes.values():
mode.boxGenerator.particleContainer = dset.particleContainer
mode.boxGenerator.offset = (offsetx, offsety)
self.filterView.update_from_dataset(dset)
self.activeMode.update_measure_viewItems()
if __name__ == '__main__':
......
from PyQt5 import QtCore, QtWidgets
from gui.filterView import FilterView, MeasureBoxGraphItem
from geometricMethods import CrossBoxSelector, SpiralSelector
from geometricMethods import BoxSelectionSubsamplingMethod, CrossBoxSubSampling, SpiralBoxSubsampling
class MeasureMode(QtCore.QObject):
def __init__(self, relatedFilterView: FilterView):
super(MeasureMode, self).__init__()
self.filterView = relatedFilterView
self.filterView: FilterView = relatedFilterView
self.uiControls: QtWidgets.QGroupBox = QtWidgets.QGroupBox()
self.boxGenerator: BoxSelectionSubsamplingMethod = None
def get_control_groupBox(self) -> QtWidgets.QGroupBox:
return self.uiControls
......@@ -15,29 +16,35 @@ class MeasureMode(QtCore.QObject):
def update_measure_viewItems(self) -> None:
raise NotImplementedError
def _send_measuredParticles_to_filterview(self) -> None:
if self.boxGenerator.particleContainer is not None:
subParticles = self.boxGenerator.apply_subsampling_method()
self.filterView.update_measured_particles(subParticles)
class CrossBoxMode(MeasureMode):
def __init__(self, *args):
super(CrossBoxMode, self).__init__(*args)
self.uiControls = CrossBoxesControls(self)
self.crossBoxGenerator: CrossBoxSelector = CrossBoxSelector(None)
self.boxGenerator: CrossBoxSubSampling = CrossBoxSubSampling(None)
self.update_measure_viewItems()
def update_measure_viewItems(self) -> None:
self.crossBoxGenerator.filterDiameter = self.filterView.filter.diameter
self.crossBoxGenerator.numBoxesAcross = int(self.uiControls.numBoxesSelector.currentText())
self.boxGenerator.filterDiameter = self.filterView.filter.diameter
self.boxGenerator.numBoxesAcross = int(self.uiControls.numBoxesSelector.currentText())
desiredCoverage: int = self.uiControls.coverageSpinbox.value()
maxCoverage: int = int(self.crossBoxGenerator.get_maximum_achievable_fraction() * 100)
maxCoverage: int = int(self.boxGenerator.get_maximum_achievable_fraction() * 100)
self.uiControls.set_to_max_possible_coverage(maxCoverage)
if desiredCoverage > maxCoverage:
desiredCoverage = maxCoverage
self.crossBoxGenerator.fraction = desiredCoverage / 100
self.boxGenerator.fraction = desiredCoverage / 100
topLefts: list = self.crossBoxGenerator.get_topLeft_of_boxes()
boxSize = self.crossBoxGenerator.boxSize
topLefts: list = self.boxGenerator.get_topLeft_of_boxes()
boxSize = self.boxGenerator.boxSize
self.filterView.update_measure_boxes(topLefts, boxSize)
self._send_measuredParticles_to_filterview()
class CrossBoxesControls(QtWidgets.QGroupBox):
......@@ -84,23 +91,19 @@ class SpiralBoxMode(MeasureMode):
def __init__(self, *args):
super(SpiralBoxMode, self).__init__(*args)
self.uiControls: SpiralBoxControls = SpiralBoxControls(self)
self.spiralBoxGenerator: SpiralSelector = SpiralSelector(None)
self.boxGenerator: SpiralBoxSubsampling = SpiralBoxSubsampling(None)
self.update_measure_viewItems()
def update_measure_viewItems(self) -> None:
self.spiralBoxGenerator.filterDiameter = self.filterView.filter.diameter
self.spiralBoxGenerator.boxSize = self.uiControls.boxSizeSpinbox.value()
# minBoxSize: float = self.spiralBoxGenerator.filterDiameter*0.1
# if self.spiralBoxGenerator.boxSize < minBoxSize:
# self.spiralBoxGenerator.boxSize = minBoxSize
# self.uiControls.boxSizeSpinbox.setValue(int(round(minBoxSize)))
self.boxGenerator.filterDiameter = self.filterView.filter.diameter
self.spiralBoxGenerator.numBoxes = self.uiControls.numBoxesSpinbox.value()
self.boxGenerator.numBoxes = self.uiControls.numBoxesSpinbox.value()
self.boxGenerator.fraction = self.uiControls.coverageSpinbox.value() / 100
topLefts: list = self.spiralBoxGenerator.get_topLeft_of_boxes()
boxSize = self.spiralBoxGenerator.boxSize
topLefts: list = self.boxGenerator.get_topLeft_of_boxes()
boxSize = self.boxGenerator.boxSize
self.filterView.update_measure_boxes(topLefts, boxSize)
self._send_measuredParticles_to_filterview()
class SpiralBoxControls(QtWidgets.QGroupBox):
......@@ -115,20 +118,23 @@ class SpiralBoxControls(QtWidgets.QGroupBox):
layout = QtWidgets.QHBoxLayout()
self.setLayout(layout)
layout.addWidget(QtWidgets.QLabel('Box Size:'))
self.boxSizeSpinbox = QtWidgets.QSpinBox()
self.boxSizeSpinbox.setValue(50)
self.boxSizeSpinbox.setMaximum(10000)
self.boxSizeSpinbox.valueChanged.connect(self._config_changed)
layout.addWidget(self.boxSizeSpinbox)
layout.addStretch()
layout.addWidget(QtWidgets.QLabel('Num Boxes:'))
self.numBoxesSpinbox = QtWidgets.QSpinBox()
self.numBoxesSpinbox.setValue(10)
self.numBoxesSpinbox.valueChanged.connect(self._config_changed)
layout.addWidget(self.numBoxesSpinbox)
layout.addStretch()
layout.addWidget(QtWidgets.QLabel('Desired Coverage (%)'))
self.coverageSpinbox = QtWidgets.QSpinBox()
self.coverageSpinbox.setFixedWidth(50)
self.coverageSpinbox.setMinimum(0)
self.coverageSpinbox.setMaximum(100)
self.coverageSpinbox.setValue(10)
self.coverageSpinbox.valueChanged.connect(self._config_changed)
layout.addWidget(self.coverageSpinbox)
def _config_changed(self):
self.measureModeParent.update_measure_viewItems()
if self.numBoxesSpinbox.value() > 0:
self.measureModeParent.update_measure_viewItems()
......@@ -32,39 +32,47 @@ class ParticleBinSorter(object):
return binIndex
def box_overlaps_contour(boxTopLeftXY: tuple, boxWidthHeight: tuple, contour, offset: tuple = (0, 0)) -> bool:
def get_Anger_fraction(numParticles, sigma=1.65, mpFraction=0.01, errorMargin=0.1):
"""
Returns the required fraction for reaching the defined errorMargin at a given number of Particles.
According to: Anger et al. "Raman microspectroscopy as a tool for microplastic particle analysis",
TrAC Trends in Analytical Chemistry, 2018, 214-226. (https://doi.org/10.1016/j.trac.2018.10.010)
:param numParticles:
:param sigma:
:param mpFraction:
:param errorMargin:
:return:
"""
N = numParticles
P = mpFraction
e = P * errorMargin
numParticlesMeasured = np.ceil(P * (1 - P) / (e**2 / sigma**2 + P*(1 - P) / N))
return np.int(numParticlesMeasured)
def box_overlaps_contour(boxTopLeftXY: tuple, boxWidthHeight: tuple, contourData: np.ndarray) -> bool:
"""
Calculates, if a contour is overlapping a box.
:param boxTopLeftXY: topLeft of Box
:param boxWidthHeight: Width and height of box
:param contour: np.ndarrayof contour data
:param offset: optional offset (x, y) of the box (i.e., the (0, 0) of the contours coord system does not match
the (0, 0) of the box coord system.
:param contourData: np.ndarrayof contour data
:return:
"""
contourPolygon = QtGui.QPolygonF()
if type(contour) == np.ndarray:
for point in contour:
contourPolygon.append(QtCore.QPointF(point[0, 0], point[0, 1]))
elif type(contour) == QtGui.QPolygonF:
contourPolygon = contour
else:
raise TypeError
boxPolygon = QtGui.QPolygonF()
boxPolygon.append(QtCore.QPointF(boxTopLeftXY[0]+offset[0], boxTopLeftXY[1]+offset[1]))
boxPolygon.append(QtCore.QPointF(boxTopLeftXY[0]+offset[0], boxTopLeftXY[1] + boxWidthHeight[1]+offset[1]))
boxPolygon.append(QtCore.QPointF(boxTopLeftXY[0]+offset[0] + boxWidthHeight[0], boxTopLeftXY[1]+offset[1]))
boxPolygon.append(QtCore.QPointF(boxTopLeftXY[0]+offset[0] + boxWidthHeight[0],
boxTopLeftXY[1] + boxWidthHeight[1]+offset[1]))
isOverlapping: bool = boxPolygon.intersects(contourPolygon)
if not isOverlapping:
# sometimes, the polygon.intersects method does not capture everything... We test the brects therefore..
polygonBrect: QtCore.QRectF = contourPolygon.boundingRect()
boxBrect: QtCore.QRectF = boxPolygon.boundingRect()
if boxBrect.contains(polygonBrect) or boxBrect.intersects(polygonBrect):
isOverlapping = True
isOverlapping: bool = False
xmin, xmax = np.min(contourData[:, 0, 0]), np.max(contourData[:, 0, 0])
width: float = xmax - xmin
boxXmin, boxXmax = boxTopLeftXY[0], boxTopLeftXY[0] + boxWidthHeight[0]
if xmin > (boxXmin-width/2):
if xmax < (boxXmax+width/2):
ymin, ymax = np.min(contourData[:, 0, 1]), np.max(contourData[:, 0, 1])
height = ymax - ymin
boxYmin, boxYmax = boxTopLeftXY[1], boxTopLeftXY[1] + boxWidthHeight[1]
if ymin > (boxYmin-height/2):
if ymax < (boxYmax+width/2):
isOverlapping = True
return isOverlapping
......
import os
import pickle
from evaluation import TotalResults
def load_results(fname: str) -> TotalResults:
res: TotalResults = None
if os.path.exists(fname):
with open(fname, "rb") as fp:
res = pickle.load(fp)
return res
return None
def save_results(fname: str, result: TotalResults) -> None:
with open(fname, "wb") as fp:
pickle.dump(result, fp, protocol=-1)
def get_pkls_from_directory(dirPath: str) -> dict:
"""
Takes a directory and finds all pkl files in there. The result is returned in a dictionary, where
each subfolder is present as a key and the actual pkl paths in a list as values.
:param dirPath:
:return:
"""
resultDict: dict = {}
subFolders = [x[1] for x in os.walk(dirPath)][0]
for subFolder in subFolders:
if subFolder.find('ignore') == -1:
subFolderPath: str = os.path.join(dirPath, subFolder)
filesInFolder: list = os.listdir(subFolderPath)
pkls: list = [os.path.join(subFolderPath, file) for file in filesInFolder if file.endswith('.pkl')]
resultDict[subFolder] = pkls
return resultDict
def get_attributes_from_foldername(foldername: str) -> list:
return [name.strip() for name in foldername.split(',')]
\ No newline at end of file
......@@ -16,60 +16,90 @@ class SubsamplingMethod(object):
self.particleContainer = particleConatainer
self.fraction = desiredFraction
def apply_subsampling_method(self) -> tuple:
@property
def label(self) -> str:
"""
A specific label that can be used for plots, for instance.
:return:
"""
raise NotImplementedError
def apply_subsampling_method(self) -> list:
"""
Takes all particles from the supplied particle conatiner and returns a new list of particles that
were measured by applying that subsampling procedure. Also, the actualy measured fraction is returned.
(The desired fraction may not always be achievable)
:returns actuallyMeasuredFraction, listOfSubParticles:
:returns listOfActuallyMeasuredParticles:
"""
raise NotImplementedError
def equals(self, otherMethod) -> bool:
"""
Checks if another provided method has the same configuration as the used instance.
:param otherMethod:
:return isEqual:
"""
raise NotImplementedError
def matches_any_pattern(self, patternList: list) -> bool:
"""
Tests. wether one of the given patterns is matching.
:param patternList:
:return:
"""
matches: bool = False
for pattern in patternList:
if self.matches_pattern(pattern):
matches = True
break
return matches
def matches_pattern(self, pattern: str) -> bool:
"""
Tests, wether the method matches a given pattern. Strings of at least 4 characters are required!
:param pattern: The string to test against
:return matchesThePattern:
"""
matches: bool = False
if len(pattern) > 3 and not pattern == 'layout':
matches = (self.label.lower().find(pattern.lower()) != -1)
return matches
class RandomSampling(SubsamplingMethod):
def apply_subsampling_method(self):
@property
def label(self) -> str:
return 'Random Subsampling'
def apply_subsampling_method(self) -> list:
numOrigParticles = len(self.particleContainer.particles)
numParticles = self._get_number_of_random_particles(numOrigParticles)
subParticles = random.sample(self.particleContainer.particles, numParticles)
return self.fraction, subParticles
return subParticles
def _get_number_of_random_particles(self, numTotalParticles):
return np.int(np.ceil(numTotalParticles * self.fraction))
class IvlevaSubsampling(SubsamplingMethod):
def __init__(self, particleContainer, sigma=1.65, mpFraction=0.01, errorMargin=0.1):
super(IvlevaSubsampling, self).__init__(particleContainer)
self.sigma = sigma
self.estimatedMPFraction = mpFraction
self.errorMargin = errorMargin
def apply_subsampling_method(self):
N = self.particleContainer.getNumberOfParticles()
numParticlesMeasured = self._get_ivleva_fraction(N)
subParticles = random.sample(self.particleContainer.particles, numParticlesMeasured)
fractionMeasured = numParticlesMeasured/N
return fractionMeasured, subParticles
def _get_ivleva_fraction(self, N):
P = self.estimatedMPFraction
e = P * self.errorMargin
numParticlesMeasured = np.ceil(P*(1 - P) / (e**2/self.sigma**2 + P*(1-P)/N))
return np.int(numParticlesMeasured)
def equals(self, otherMethod) -> bool:
return type(otherMethod) == type(self) and otherMethod.fraction == self.fraction
class SizeBinFractioning(SubsamplingMethod):
def __init__(self, particleConatiner, desiredfraction: float = 0.2):
super(SizeBinFractioning, self).__init__(particleConatiner, desiredfraction)
self.sorter: ParticleBinSorter = ParticleBinSorter()
def apply_subsampling_method(self):
@property
def label(self) -> str:
return 'SizeBin Subsampling'
def apply_subsampling_method(self) -> list:
subParticlesPerBin: list = self._get_subParticles_per_bin(self.particleContainer.particles)
subParticles: list = []
for subParticleList in subParticlesPerBin:
for particle in subParticleList:
subParticles.append(particle)
return self.fraction, subParticles
return subParticles
def _get_subParticles_per_bin(self, particleList: list):
particlesInBins: list = self.sorter.sort_particles_into_bins(particleList)
......@@ -84,3 +114,6 @@ class SizeBinFractioning(SubsamplingMethod):
subParticlesPerBin.append(subParticlesInBin)
return subParticlesPerBin
def equals(self, otherMethod) -> bool:
return type(otherMethod) == type(self) and otherMethod.fraction == self.fraction
import numpy as np
import matplotlib.pyplot as plt
import time
import sys
sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset
from methods import IvlevaSubsampling, RandomSampling, SizeBinFractioning
from geometricMethods import BoxSelectionCreator
from helpers import ParticleBinSorter
from evaluation import ResultComparer
from evaluation import TotalResults, SampleResult
from input_output import get_pkls_from_directory, get_attributes_from_foldername, save_results, load_results
fname: str = r'C:\Users\xbrjos\Desktop\temp MP\190313_Soil_5_A_50_5_1_50_1\190313_Soil_5_A_50_5_1_50_1.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\190326_MCII_WWTP_SB_50_2\190326_MCII_WWTP_SB_50_2.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\190326_MCII_WWTP_SB_50_1\190326_MCII_WWTP_SB_50_1.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl' #legacy convert not working..
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\190201_BSB_Stroomi_ds2_R1_R2_50\190201_BSB_Stroomi_ds2_R1_R2_50.pkl' #zvalues image missing, legacy convert fails..
"""
IMPORTANT!!!
SET GEPARD TO EVALUATION BRANCH (WITHOUT THE TILING STUFF), OTHERWISE SOME OF THE LEGACY CONVERTS MIGHT FAIL..
"""
dset = dataset.loadData(fname)
print('loaded dataset')
boxCreator = BoxSelectionCreator(dset)
center, size = boxCreator.get_filterDimensions_from_dataset()
print(center, size)
print(dset.mapToPixel(center, force=True))
# pc = dset.particleContainer
# origParticles = pc.particles
# resultComparer = ResultComparer()
# numOrigMP = resultComparer._get_number_of_MP_particles(origParticles)
# print(f'orig particles: {len(origParticles)}, of which are mp: {numOrigMP}')
# # ivlevaSampling = IvlevaSubsampling(pc)
# # ivlevaFraction, ivlevaParticles = ivlevaSampling.apply_subsampling_method()
# results: TotalResults = TotalResults()
# pklsInFolders = get_pkls_from_directory(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets')
#
# for folder in pklsInFolders.keys():
# for samplePath in pklsInFolders[folder]:
# newSampleResult: SampleResult = results.add_sample(samplePath)
# for attr in get_attributes_from_foldername(folder):
# newSampleResult.set_attribute(attr)
#
# t0 = time.time()
# fractions = np.arange(0.05, .55, 0.05)
# errors = []
# binErrors = []
# numIterations = 1000
# for fraction in fractions:
# print('random sampling, fraction:', fraction)
# # randomSampling = RandomSampling(pc, desiredFraction=fraction)
# randomSampling = SizeBinFractioning(pc, fraction)
# iterErrors = []
# binIterErrors = []
# for _ in range(numIterations):
# randomFraction, randomParticles = randomSampling.apply_subsampling_method()
# iterErrors.append(resultComparer._get_mp_count_error(origParticles, randomParticles, randomFraction))
# bins, errorsPerBin = resultComparer._get_mp_count_error_per_bin(origParticles, randomParticles, randomFraction)
# binIterErrors.append(errorsPerBin)
# errors.append(round(np.mean(iterErrors)*100)) #from fraction to %
# fractionBinErrors = []
# for binIndex in range(len(bins)+1):
# binError = round(np.mean([binIterErrors[i][binIndex] for i in range(numIterations)]) * 100)
# fractionBinErrors.append(binError)
# binErrors.append(fractionBinErrors)
# print('random sampling took', np.round(time.time()-t0, 2), 'seonds')
# binLowerLimits = bins.copy()
# binLowerLimits.insert(0, 0)
# plt.subplot(121)
# plt.plot(fractions, errors)
# # plt.title(f'Random Sampling, averaged from {numIterations} trials, orig particle count: {len(origParticles)}')
# plt.xlabel('Fraction measured')
# plt.ylabel('Average error in MP particle count (%)')
# plt.subplot(122)
# for fracMeas, curBinErrors in zip(fractions, binErrors):
# plt.plot(binLowerLimits, curBinErrors, label=np.round(fracMeas, 1))
# # plt.title('Error in MP count (%) per size bin')
# plt.xlabel('particle size')
# plt.ylabel('Average error in MP particle count (%)')
# plt.legend()
# plt.show()
# results.update_all()
# print('updating all took', time.time()-t0, 'seconds')
#
# save_results('results1.res', results)
results: TotalResults = load_results('results1.res')
errorPerFraction: dict = results.get_error_vs_fraction_data(methods=['spiral', 'cross'])
plt.clf()
for methodLabel in errorPerFraction.keys():
fractions: list = list(errorPerFraction[methodLabel].keys())
errors: list = list(errorPerFraction[methodLabel].values())
plt.plot(fractions, errors, label=methodLabel)
plt.title('Spiral or Box Layouts')
plt.xscale('log')
plt.xlabel('measured fraction')
plt.ylabel('mpCountError')
plt.legend()
plt.show()
# # sizeBinSampling = SizeBinFractioning(pc)
# # sizeBinParticles = sizeBinSampling.apply_subsampling_method()
\ No newline at end of file
......@@ -13,52 +13,288 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
import gepard
from gepard.analysis.particleAndMeasurement import Particle, Measurement
from evaluation import ResultComparer
from evaluation import TotalResults, SampleResult, SubsamplingResult
import methods as meth
import geometricMethods as gmeth
class TestResultComparer(unittest.TestCase):
class TestTotalResults(unittest.TestCase):
def setUp(self) -> None:
self.totalResults = TotalResults()
def test_add_sample(self):
newResult: SampleResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile.pkl')
self.assertEqual(len(self.totalResults.sampleResults), 1)
self.assertTrue(type(newResult) == SampleResult)
newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile.pkl') # the same file should not be added again
self.assertEqual(len(self.totalResults.sampleResults), 1)
self.assertTrue(newResult is None)
newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile2.pkl') # another should be added, though
self.assertEqual(len(self.totalResults.sampleResults), 2)
self.assertTrue(type(newResult) == SampleResult)
newResult = self.totalResults.add_sample('fakePath/fakeFolder/fakeFile2.txt') # invalid extention, not added...
self.assertEqual(len(self.totalResults.sampleResults), 2)
self.assertTrue(newResult is None)
def test_get_methods_for_fraction(self):
def containsMethod(listOfMethods: list, template: meth.SubsamplingMethod) -> bool:
contains: bool = False
for method in listOfMethods:
if type(method) == type(template) and method.fraction == template.fraction:
contains = True
break
return contains
dset: gepard.dataset.DataSet = gepard.dataset.DataSet('fakepath')
imgdim = 10
dset.imagescanMode = 'df'
dset.imagedim_df = [imgdim, imgdim]
dset.pixelscale_df = 1.0
minX, maxX, minY, maxY = 0, 1000, 0, 1000
dset.maxdim = minX + imgdim / 2, maxY - imgdim / 2, maxX - imgdim / 2, minY + imgdim / 2
desiredFraction = 0.1
methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
possibleRandomMethods = 2
possibleCrossBoxMethods = 2
possibleSpiralBoxMethods = 3
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
self.assertEqual(len(methods), totalPossible)
self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))
desiredFraction = 0.5
methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
possibleRandomMethods = 2
possibleCrossBoxMethods = 1
possibleSpiralBoxMethods = 0
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
self.assertEqual(len(methods), totalPossible)
self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
self.assertFalse(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))
desiredFraction = 0.9
methods = self.totalResults._get_methods_for_fraction(dset, desiredFraction)
possibleRandomMethods = 2
possibleCrossBoxMethods = 0
possibleSpiralBoxMethods = 0
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + possibleSpiralBoxMethods
self.assertEqual(len(methods), totalPossible)
self.assertTrue(containsMethod(methods, meth.RandomSampling(dset, desiredFraction)))
self.assertTrue(containsMethod(methods, meth.SizeBinFractioning(dset, desiredFraction)))
self.assertFalse(containsMethod(methods, gmeth.CrossBoxSubSampling(dset, desiredFraction)))
self.assertFalse(containsMethod(methods, gmeth.SpiralBoxSubsampling(dset, desiredFraction)))
def test_get_error_vs_fraction_data(self):
firstSample: SampleResult = self.totalResults.add_sample('sample1.pkl')
firstSample.set_attribute('to be used')
secondSample: SampleResult = self.totalResults.add_sample('sample2.pkl')
secondSample.set_attribute('not to be used')
firstMethod: meth.RandomSampling = meth.RandomSampling(None, 0.1)
firstResult: SubsamplingResult = SubsamplingResult(firstMethod)
firstResult.mpCountError = 0.8
secondMethod: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
secondMethod.numBoxesAcross = 3
secondResult: SubsamplingResult = SubsamplingResult(secondMethod)
secondResult.mpCountError = 0.6
thirdMethod: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
thirdMethod.numBoxesAcross = 5
self.assertEqual(thirdMethod.fraction, 0.1)
thirdResult: SubsamplingResult = SubsamplingResult(thirdMethod)
thirdResult.mpCountError = 0.4
thirdMethod2: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.1)
thirdMethod2.numBoxesAcross = 5
self.assertEqual(thirdMethod2.fraction, 0.1)
thirdResult2: SubsamplingResult = SubsamplingResult(thirdMethod)
thirdResult2.mpCountError = 0.8
thirdMethod3: gmeth.CrossBoxSubSampling = gmeth.CrossBoxSubSampling(None, 0.2)
thirdMethod3.numBoxesAcross = 5
self.assertEqual(thirdMethod3.fraction, 0.2)
thirdResult3: SubsamplingResult = SubsamplingResult(thirdMethod3)
thirdResult3.mpCountError = 0.5
firstSample.results = [firstResult, secondResult, thirdResult, thirdResult3]
secondSample.results = [firstResult, secondResult, thirdResult2, thirdResult3]
resultDict: dict = self.totalResults.get_error_vs_fraction_data()
self.assertEqual(list(resultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
for i in range(3):
res: dict = list(resultDict.values())[i]
if i == 0:
self.assertEqual(list(res.keys()), [0.1])
self.assertAlmostEqual(res[0.1], 0.8)
if i == 1:
self.assertEqual(list(res.keys()), [0.1])
self.assertAlmostEqual(res[0.1], 0.6)
if i == 2:
self.assertEqual(list(res.keys()), [0.1, 0.2])
self.assertAlmostEqual(res[0.1], 0.6) # i.e., mean([0.4, 0.8])
self.assertAlmostEqual(res[0.2], 0.5)
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(attributes=['to be used'])
self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
for i in range(3):
res: dict = list(filteredResultDict.values())[i]
if i == 0:
self.assertEqual(list(res.keys()), [0.1])
self.assertAlmostEqual(res[0.1], 0.8)
if i == 1:
self.assertEqual(list(res.keys()), [0.1])
self.assertAlmostEqual(res[0.1], 0.6)
if i == 2:
self.assertEqual(list(res.keys()), [0.1, 0.2])
self.assertAlmostEqual(res[0.1], 0.4) # only the result from the first sample is used, as filtered..
self.assertAlmostEqual(res[0.2], 0.5)
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['cross'])
self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['Cross'])
self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['random'])
self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label])
class TestSampleResult(unittest.TestCase):
def setUp(self) -> None:
self.sampleResult: SampleResult = SampleResult('fakePath/fakeFile.pkl')
self.sampleResult.dataset = gepard.dataset.DataSet('fakePath/fakeFile.pkl')
self.sampleResult.results.append(SubsamplingResult(meth.RandomSampling(None, 0.1)))
newMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
newMethod.numBoxes = 10
self.sampleResult.results.append(SubsamplingResult(newMethod))
newMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
newMethod.numBoxes = 15
self.sampleResult.results.append(SubsamplingResult(newMethod))
newMethod = gmeth.SpiralBoxSubsampling(None, 0.3)
newMethod.numBoxes = 10
self.sampleResult.results.append(SubsamplingResult(newMethod))
def test_sampleResults_added_correctly(self):
method: meth.SubsamplingMethod = self.sampleResult.results[0].method
self.assertEqual(type(method), meth.RandomSampling)
self.assertEqual(method.fraction, 0.1)
method: meth.SubsamplingMethod = self.sampleResult.results[1].method
self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
self.assertEqual(method.fraction, 0.1)
self.assertEqual(method.numBoxes, 10)
method: meth.SubsamplingMethod = self.sampleResult.results[2].method
self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
self.assertEqual(method.fraction, 0.1)
self.assertEqual(method.numBoxes, 15)
method: meth.SubsamplingMethod = self.sampleResult.results[3].method
self.assertEqual(type(method), gmeth.SpiralBoxSubsampling)
self.assertEqual(method.fraction, 0.3)
self.assertEqual(method.numBoxes, 10)
def test_result_is_already_present(self):
newMethod: meth.SubsamplingMethod = meth.RandomSampling(None, 0.1)
self.assertTrue(self.sampleResult._result_is_already_present(newMethod))
newMethod: meth.SubsamplingMethod = meth.RandomSampling(None, 0.2)
self.assertFalse(self.sampleResult._result_is_already_present(newMethod))
newMethod: meth.SubsamplingMethod = gmeth.SpiralBoxSubsampling(None, 0.1)
self.assertTrue(self.sampleResult._result_is_already_present(newMethod))
newMethod: meth.SubsamplingMethod = gmeth.SpiralBoxSubsampling(None, 0.2)
self.assertFalse(self.sampleResult._result_is_already_present(newMethod))
newMethod: meth.SubsamplingMethod = gmeth.CrossBoxSubSampling(None, 0.3)
self.assertFalse(self.sampleResult._result_is_already_present(newMethod))
def test_remove_result_of_method(self):
numOrigResults = len(self.sampleResult.results)
self.sampleResult._remove_result_of_method(meth.RandomSampling(None, 0.1))
self.assertEqual(len(self.sampleResult.results), numOrigResults-1)
self.sampleResult._remove_result_of_method(gmeth.SpiralBoxSubsampling(None, 0.1))
self.assertEqual(len(self.sampleResult.results), numOrigResults-2)
self.sampleResult._remove_result_of_method(gmeth.SpiralBoxSubsampling(None, 0.2)) # this is one is not present...
self.assertEqual(len(self.sampleResult.results), numOrigResults-2)
def test_attributes(self):
self.sampleResult.set_attribute('soil')
self.assertTrue(self.sampleResult.has_attribute('soil'))
self.assertTrue(self.sampleResult.has_attribute('Soil')) # we want to be case insensitive
self.assertTrue(self.sampleResult.has_attribute('SOIL'))