...
 
Commits (2)
...@@ -15,11 +15,11 @@ sys.path.append("C://Users//xbrjos//Desktop//Python") ...@@ -15,11 +15,11 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard.analysis.particleContainer import ParticleContainer from gepard.analysis.particleContainer import ParticleContainer
from gepard.analysis import particleAndMeasurement as pm from gepard.analysis import particleAndMeasurement as pm
from gepard.analysis import particleCharacterization as pc from gepard.analysis import particleCharacterization as pc
from gepard.helperfunctions import cv2imread_fix
from methods import SubsamplingMethod from methods import SubsamplingMethod
from helpers import timingDecorator from helpers import timingDecorator
def get_pca(data: np.ndarray, numComp: int = 2) -> np.ndarray: def get_pca(data: np.ndarray, numComp: int = 2) -> np.ndarray:
try: try:
standardizedData = StandardScaler().fit_transform(data.copy()) standardizedData = StandardScaler().fit_transform(data.copy())
...@@ -61,34 +61,57 @@ def get_n_points_closest_to_point(points: np.ndarray, n: int, refPoint: np.ndarr ...@@ -61,34 +61,57 @@ def get_n_points_closest_to_point(points: np.ndarray, n: int, refPoint: np.ndarr
return list(sortedIndices[:n]) return list(sortedIndices[:n])
def get_particle_featurematrix(particleContainer: ParticleContainer) -> np.ndarray: def get_particle_featurematrix(particleContainer: ParticleContainer, fullimg: np.ndarray = None) -> np.ndarray:
""" """
:return: np.ndarray, numRows: Features, numCols: Particles :return: np.ndarray, numRows: Features, numCols: Particles
""" """
vectors: list = [] vectors: list = []
for particle in particleContainer.particles: firstVecLength: int = 0
vectors.append(get_characteristic_vector(particle)) for index, particle in enumerate(particleContainer.particles):
particleImg: np.ndarray = None
if fullimg is not None:
particleImg = pc.getParticleImageFromFullimage(particle.contour, fullimg)
vector: np.ndarray = get_characteristic_vector(particle, particleImg)
if index == 0:
firstVecLength = vector.shape[0]
else:
assert vector.shape[0] == firstVecLength, 'particle feature vectors have non-uniform lengths...'
vectors.append(vector)
vectors: np.ndarray = np.array(vectors) vectors: np.ndarray = np.array(vectors)
assert vectors.shape[0] == len(particleContainer.particles) assert vectors.shape[0] == len(particleContainer.particles)
return vectors return vectors
def get_characteristic_vector(particle: pm.Particle) -> np.ndarray: def get_characteristic_vector(particle: pm.Particle, particleImg: np.ndarray = None) -> np.ndarray:
vector: list = [] vector: list = []
# vector += list(get_log_hu_moments(particle.contour)) vector += list(get_log_hu_moments(particle.contour))
vector.append(float(get_color_index(particle.color))) vector.append(float(get_color_index(particle.color)))
vector.append(get_solidity(particle.contour)) vector.append(get_solidity(particle.contour))
vector.append(get_aspect_ratio(particle.contour)) vector.append(get_aspect_ratio(particle.contour))
vector.append(get_extent(particle.contour)) vector.append(get_extent(particle.contour))
vector.append(cv2.contourArea(particle.contour)) vector.append(cv2.contourArea(particle.contour))
# vector.append(get_shape_index(particle.shape)) vector.append(get_shape_index(particle.shape))
# vector.append(cv2.arcLength(particle.contour, True)) vector.append(cv2.arcLength(particle.contour, True))
for ftHarmonic in get_curvature_ft(particle.contour):
vector.append(ftHarmonic)
if particleImg is not None:
image_features: np.ndarray = get_image_feature_vec(particleImg)
for val in image_features:
vector.append(val)
# vector: np.ndarray = np.hstack((log_hu, color)) # vector: np.ndarray = np.hstack((log_hu, color))
# if len(vector) != 11: # if len(vector) != 11:
# print('error') # print('error')
# assert len(vector) == 7 + 4, f'wrong feature vector: {vector} with shape: {vector.shape}' # assert len(vector) == 7 + 4, f'wrong feature vector: {vector} with shape: {vector.shape}'
for entry in vector:
try:
float(entry)
except ValueError:
print('not numeric value found')
raise
assert not np.isnan(entry)
return np.array(vector) return np.array(vector)
...@@ -141,6 +164,23 @@ def get_log_hu_moments(contour: np.ndarray) -> np.ndarray: ...@@ -141,6 +164,23 @@ def get_log_hu_moments(contour: np.ndarray) -> np.ndarray:
return resultMoments[:, 0] return resultMoments[:, 0]
def get_image_feature_vec(particleImg: np.ndarray) -> np.ndarray:
meanStdev: np.ndarray = get_mean_and_stdev(particleImg)
return meanStdev.reshape(meanStdev.size)
def get_mean_and_stdev(img: np.ndarray) -> np.ndarray:
try:
meanStd: tuple = cv2.meanStdDev(img)
colorMean: np.ndarray = np.array([i[0] for i in meanStd[0]])
colorStd: np.ndarray = np.array([i[0] for i in meanStd[1]])
except cv2.error: # i.e, one pixel images...
colorMean: np.ndarray = np.array([128, 128, 128])
colorStd: np.ndarray = np.array([0, 0, 0])
# print('invalid here...:', img, img.shape, colorMean, colorStd, np.vstack((colorMean, colorStd)))
return np.vstack((colorMean, colorStd))
def get_color_hash(color: str, desiredLength: int = 4) -> np.ndarray: def get_color_hash(color: str, desiredLength: int = 4) -> np.ndarray:
colorArray: list = [int(i) for i in str(abs(hash(color)))[:desiredLength]] colorArray: list = [int(i) for i in str(abs(hash(color)))[:desiredLength]]
return np.transpose(np.array(colorArray)) return np.transpose(np.array(colorArray))
...@@ -153,25 +193,72 @@ def get_color_index(color: str) -> int: ...@@ -153,25 +193,72 @@ def get_color_index(color: str) -> int:
return colors.index(color) return colors.index(color)
# def get_shape_index(shape: str) -> int: def get_shape_index(shape: str) -> int:
# shapes: list = ['spherule', 'fibre', 'flake', 'irregular'] shapes: list = ['spherule', 'fibre', 'flake', 'irregular', None]
# assert shape in shapes assert shape in shapes
# return shapes.index(shape) return shapes.index(shape)
def get_curvature_ft(contour: np.ndarray, angularSegment: float = 20, numHarmonics: int = 8) -> np.ndarray:
"""
Calculates curvature of the contour and applies discrete fourier transform (dft) to it.
Adapted from Xu et al, "Comparison of shape features for the classification of wear particles",
doi:10.1016/S0952-1976(97)00017-1
:param angularSegment: In degree, section of 360 degree to consider for each angle
:param contour: Particle Contour
:param numHarmonics: Number of FT Harmonics to return
:return: the computed
"""
m: int = int(round(contour.shape[0] / (360 / angularSegment))) # smoothing factor, is this reasonable? That's, how it's described in the paper...
numPoints: int = contour.shape[0]
curvature: np.ndarray = np.zeros(numPoints)
for i in range(numPoints):
x, y = contour[i, 0, 0], contour[i, 0, 1]
index_before = i-m-1
index_after = i+m+1
if index_after > numPoints-1:
index_after -= numPoints
x_before, y_before = contour[index_before, 0, 0], contour[index_before, 0, 1]
x_after, y_after = contour[index_after, 0, 0], contour[index_after, 0, 1]
tan_before: float = 0
if x - x_before != 0:
tan_before = np.rad2deg(np.arctan((y-y_before) / (x-x_before)))
tan_after: float = 0
if x_after - x != 0:
tan_after: float = np.rad2deg(np.arctan((y_after-y) / (x_after-x)))
curvature[i] = tan_after - tan_before
dft: np.ndarray = np.fft.fft(curvature)
freqs: np.ndarray = np.fft.fftfreq(numPoints)
mask: np.ndarray = freqs > 0
dft_true: np.ndarray = 2 * np.abs(dft/numPoints)
dft_true = dft_true[mask]
numFTPoints = dft_true.shape[0]
# let's only take the odd ones (they seem to carry more information).
# Also, make sure that always numHarmonics numbers come out. Fill up with zeros, if contour was very short
indicesToTake: np.ndarray = np.arange(numHarmonics)*2 + 1
final_dfts = np.zeros(numHarmonics)
for i, indexToTake in enumerate(indicesToTake):
if indexToTake < numFTPoints:
final_dfts[i] = dft_true[indexToTake]
return final_dfts
class TrainedSubsampling(SubsamplingMethod): class TrainedSubsampling(SubsamplingMethod):
def __init__(self, particleContainer: ParticleContainer, desiredFraction: float, def __init__(self, particleContainer: ParticleContainer, desiredFraction: float,
path: str = r'C:\Users\xbrjos\Desktop\Python\Subsampling\chemometrics\RandomForestClassifier, score 0.72.pkl'): path: str = r'C:\Users\xbrjos\Desktop\Python\Subsampling\chemometrics\RandomForestClassifier, score 0.79.pkl'):
super(TrainedSubsampling, self).__init__(particleContainer, desiredFraction) super(TrainedSubsampling, self).__init__(particleContainer, desiredFraction)
self.score: float = None self.score: float = None
self.clf = None self.clf = None
self.clfPath: str = path self.clfPath: str = path
self.fraction = desiredFraction self.fraction = desiredFraction
# @property
# def fraction(self) -> float:
# return self.desiredFraction/2
def equals(self, otherMethod) -> bool: def equals(self, otherMethod) -> bool:
isEqual: bool = False isEqual: bool = False
if type(otherMethod) == TrainedSubsampling and otherMethod.fraction == self.fraction: if type(otherMethod) == TrainedSubsampling and otherMethod.fraction == self.fraction:
...@@ -189,7 +276,12 @@ class TrainedSubsampling(SubsamplingMethod): ...@@ -189,7 +276,12 @@ class TrainedSubsampling(SubsamplingMethod):
def apply_subsampling_method(self) -> list: def apply_subsampling_method(self) -> list:
self._load_classifier() self._load_classifier()
features: np.ndarray = get_particle_featurematrix(self.particleContainer) fullimgpath: str = r'C:\Users\xbrjos\Desktop\temp MP\Fullimages'
dsetname: str = self.particleContainer.datasetParent.name
imgPath: str = os.path.join(fullimgpath, dsetname + '.tif')
fullimg = cv2imread_fix(imgPath)
features: np.ndarray = get_particle_featurematrix(self.particleContainer, fullimg)
predictions: np.ndarray = self.clf.predict(features) predictions: np.ndarray = self.clf.predict(features)
indicesToSelect: set = self._get_measure_indices(list(predictions)) indicesToSelect: set = self._get_measure_indices(list(predictions))
selectedParticles: list = [] selectedParticles: list = []
...@@ -199,9 +291,6 @@ class TrainedSubsampling(SubsamplingMethod): ...@@ -199,9 +291,6 @@ class TrainedSubsampling(SubsamplingMethod):
return selectedParticles return selectedParticles
# def _make_subparticles_match_fraction(self, subParticles: list) -> list:
# return subParticles
def _load_classifier(self) -> None: def _load_classifier(self) -> None:
assert os.path.exists(self.clfPath) assert os.path.exists(self.clfPath)
fname: str = self.clfPath fname: str = self.clfPath
......
...@@ -12,9 +12,12 @@ from sklearn.feature_selection import chi2 ...@@ -12,9 +12,12 @@ from sklearn.feature_selection import chi2
import pickle import pickle
import time import time
import sys import sys
import os
sys.path.append("C://Users//xbrjos//Desktop//Python") sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset from gepard import dataset
from gepard.analysis.particleContainer import ParticleContainer from gepard.analysis.particleContainer import ParticleContainer
from gepard.analysis.particleCharacterization import getParticleImageFromFullimage
from gepard.helperfunctions import cv2imread_fix
from input_output import get_pkls_from_directory from input_output import get_pkls_from_directory
from chemometricMethods import get_log_hu_moments, get_color_index, get_pca, get_characteristic_vector from chemometricMethods import get_log_hu_moments, get_color_index, get_pca, get_characteristic_vector
...@@ -58,41 +61,52 @@ def test_classification_models(dataset: tuple) -> None: ...@@ -58,41 +61,52 @@ def test_classification_models(dataset: tuple) -> None:
if __name__ == '__main__': if __name__ == '__main__':
recreateNew: bool = False recreateNew: bool = True
if recreateNew: if recreateNew:
pklsInFolders: dict = get_pkls_from_directory(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets') pklsInFolders: dict = get_pkls_from_directory(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets')
fullimgpath: str = r'C:\Users\xbrjos\Desktop\temp MP\Fullimages'
X: list = [] X: list = []
y: list = [] y: list = []
counter = 0 counter = 0
for folder in pklsInFolders.keys(): for folder in pklsInFolders.keys():
for pklPath in pklsInFolders[folder]: for pklPath in pklsInFolders[folder]:
if counter < 100: if counter < 50:
dset: dataset.DataSet = dataset.loadData(pklPath) dset: dataset.DataSet = dataset.loadData(pklPath)
print('loaded', dset.name) print('loaded', dset.name, counter)
imgPath: str = os.path.join(fullimgpath, dset.name + '.tif')
fullimg = cv2imread_fix(imgPath)
print('loaded fullimg', imgPath, counter)
partContainer: ParticleContainer = dset.particleContainer partContainer: ParticleContainer = dset.particleContainer
for particle in partContainer.particles:
features: np.ndarray = get_characteristic_vector(particle) firstVecLength: int = 0
# features: list = [abs(i) for i in get_log_hu_moments(particle.contour)] for index, particle in enumerate(partContainer.particles):
# features.append(get_color_index(particle.color)) partImg: np.ndarray = getParticleImageFromFullimage(particle.contour, fullimg)
features: np.ndarray = get_characteristic_vector(particle, partImg)
if index == 0:
firstVecLength = features.shape[0]
else:
assert features.shape[0] == firstVecLength
X.append(features) X.append(features)
y.append(int(is_MP_particle(particle))) y.append(int(is_MP_particle(particle)))
counter += 1 counter += 1
X: np.ndarray = np.array(X) X_all: np.ndarray = np.array(X)
y: np.ndarray = np.array(y) y_all: np.ndarray = np.array(y)
MPindices: np.ndarray = np.where(y == 1)[0] X_all: np.ndarray = SelectKBest(chi2, k=15).fit_transform(abs(X_all), y_all)
nonMPindices: np.ndarray = np.where(y == 0)[0]
MPindices: np.ndarray = np.where(y_all == 1)[0]
nonMPindices: np.ndarray = np.where(y_all == 0)[0]
nonMPindices: list = sample(list(nonMPindices), len(MPindices)) nonMPindices: list = sample(list(nonMPindices), len(MPindices))
X_MP: list = list(X[MPindices]) X_MP: list = list(X_all[MPindices])
y_MP: list = list(y[MPindices]) y_MP: list = list(y_all[MPindices])
X_nonMP: list = list(X[nonMPindices]) X_nonMP: list = list(X_all[nonMPindices])
y_nonMP: list = list(y[nonMPindices]) y_nonMP: list = list(y_all[nonMPindices])
assert set(y_MP) == {1} assert set(y_MP) == {1}
assert set(y_nonMP) == {0} assert set(y_nonMP) == {0}
...@@ -100,6 +114,7 @@ if __name__ == '__main__': ...@@ -100,6 +114,7 @@ if __name__ == '__main__':
X_equalized: np.ndarray = np.array(X_MP + X_nonMP) X_equalized: np.ndarray = np.array(X_MP + X_nonMP)
y_equalized: np.ndarray = np.array(y_MP + y_nonMP) y_equalized: np.ndarray = np.array(y_MP + y_nonMP)
sum = X_MP + X_nonMP
dset: tuple = (X_equalized, y_equalized) dset: tuple = (X_equalized, y_equalized)
...@@ -111,20 +126,36 @@ if __name__ == '__main__': ...@@ -111,20 +126,36 @@ if __name__ == '__main__':
X, y = dset X, y = dset
with open(r'C:\Users\xbrjos\Desktop\Python\Subsampling\chemometrics\RandomForestClassifier, score 0.72.pkl', "rb") as fp: # with open(r'C:\Users\xbrjos\Desktop\Python\Subsampling\chemometrics\RandomForestClassifier, score 0.72.pkl', "rb") as fp:
clf: RandomForestClassifier = pickle.load(fp) # clf: RandomForestClassifier = pickle.load(fp)
y_predicted = clf.predict(X) # y_predicted = clf.predict(X)
# np.savetxt('Data.txt', X) np.savetxt('Data.txt', X)
# np.savetxt('Assignments.txt', y) np.savetxt('Assignments.txt', y)
np.savetxt('Data_all.txt', X_all)
np.savetxt('Assignments_all.txt', y_all)
# princComps = get_pca(X.transpose(), numComp=2) # princComps = get_pca(X.transpose(), numComp=2)
# #
# plt.scatter(princComps[:, 0], princComps[:, 1]) # plt.scatter(princComps[:, 0], princComps[:, 1])
# print(X_equalized.shape) # print(X_equalized.shape)
# X: np.ndarray = SelectKBest(chi2, k=5).fit_transform(X, y) # X: np.ndarray = SelectKBest(chi2, k=15).fit_transform(X, y)
# print(X_equalized.shape) # print(X_equalized.shape)
# test_classification_models((X, y)) test_classification_models((X, y))
X = StandardScaler().fit_transform(X)
clf = RandomForestClassifier(n_estimators=1000)
clf.fit(X, y)
score = clf.score(X_all, y_all)
y_predicted = clf.predict(X_all)
errors: dict = {int(k): 0 for k in np.unique(y_all)}
for j in range(len(y_predicted)):
if y_all[j] != y_predicted[j]:
errors[y_all[j]] += 1
print('num MP Particles in set:', len(X_MP))
print(f'randForest with test size {len(y_all)} has score {round(score, 2)}, errors: {errors}')
import sys
sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard.helperfunctions import cv2imread_fix
from gepard.dataset import loadData
from gepard.analysis.particleContainer import ParticleContainer
import cv2
import numpy as np
from scipy import spatial
import os
import matplotlib.pyplot as plt
from helpers import get_filterDimensions_from_dataset, get_center_from_filter_dimensions, convert_length_to_pixels
from graphs import get_distance_point_histogramdata
if __name__ == '__main__':
# imgpath: str = r'C:\Users\xbrjos\Desktop\temp MP\Fullimages'
# imgname: str = '181120_MCI_2_ds1+2_all_ kleiner500_10_1.tif'
# imgname: str = '190619_5_PTPH_sld_190321_ds1_50_1_neu.tif'
#
# img: np.ndarray = cv2imread_fix(os.path.join(imgpath, imgname))
# gray: np.ndarray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# _, binimg = cv2.threshold(gray, 20, 1, cv2.THRESH_BINARY_INV)
# distmap: np.ndarray = cv2.distanceTransform(binimg, cv2.DIST_L1, 3)
# plt.imshow(distmap, cmap='gray')
paths: list = [r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets\water\181120_MCI_2_ds1+2_all_ kleiner500_10_1.pkl',
r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets\wastewater, slush\190619_5_PTPH_sld_190321_ds1_50_1_neu.pkl']
# paths.append(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets\water\190306_MCII_1_2_50.pkl')
# paths.append(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets\water\190222_MCII_1_1_50_1.pkl')
distances: list = []
for path in paths:
dset = loadData(path)
particleContainer: ParticleContainer = dset.particleContainer
# particleCenters: list = []
# for particle in particleContainer.particles:
# particleCenters.append([np.mean(particle.contour[:, 0, 0]), np.mean(particle.contour[:, 0, 1])])
#
# closest_particle_distances: np.ndarray = np.zeros(len(particleCenters))
# particleCenters: np.ndarray = np.array(particleCenters)
# print('particle centers done')
# distMat: np.ndarray = spatial.distance_matrix(particleCenters, particleCenters)
# print('distmat computed')
# for i in range(distMat.shape[0]):
# if i == 0:
# closest_particle_distances[i] = np.min(distMat[i, 1:])
# elif i == distMat.shape[0]-1:
# closest_particle_distances[i] = np.min(distMat[i, :-1])
# else:
# closest_particle_distances[i] = np.min([np.min(distMat[i, :i]), np.min(distMat[i, i+1:])])
# distances.append(closest_particle_distances)
# plt.boxplot(distances)
...@@ -5,11 +5,12 @@ Created on Wed Jan 22 13:57:28 2020 ...@@ -5,11 +5,12 @@ Created on Wed Jan 22 13:57:28 2020
@author: luna @author: luna
""" """
import pickle # import pickle
import os import os
import numpy as np import numpy as np
import matplotlib.pyplot as plt # import matplotlib.pyplot as plt
import concurrent.futures import concurrent.futures
import operator
import sys import sys
sys.path.append("C://Users//xbrjos//Desktop//Python") sys.path.append("C://Users//xbrjos//Desktop//Python")
from gepard import dataset from gepard import dataset
...@@ -32,8 +33,8 @@ def get_methods_to_test(dataset: dataset.DataSet, fractions: list = [], maxTries ...@@ -32,8 +33,8 @@ def get_methods_to_test(dataset: dataset.DataSet, fractions: list = [], maxTries
:return: list of measurement Objects that are applicable :return: list of measurement Objects that are applicable
""" """
if len(fractions) == 0: if len(fractions) == 0:
# fractions: list = [0.02, 0.05, 0.1, 0.25, 0.5, 0.7, 0.9] fractions: list = [0.02, 0.05, 0.1, 0.25, 0.5, 0.7, 0.9]
fractions: list = [0.1, 0.3, 0.5] # fractions: list = [0.1, 0.3, 0.5]
methods: list = [] methods: list = []
particleContainer = dataset.particleContainer particleContainer = dataset.particleContainer
...@@ -46,7 +47,7 @@ def get_methods_to_test(dataset: dataset.DataSet, fractions: list = [], maxTries ...@@ -46,7 +47,7 @@ def get_methods_to_test(dataset: dataset.DataSet, fractions: list = [], maxTries
methods += boxCreator.get_spiralBoxSubsamplers_for_fraction(fraction) methods += boxCreator.get_spiralBoxSubsamplers_for_fraction(fraction)
methods += boxCreator.get_randomBoxSubsamplers_for_fraction(fraction, maxTries=maxTries) methods += boxCreator.get_randomBoxSubsamplers_for_fraction(fraction, maxTries=maxTries)
methods += boxCreator.get_randomQuarterBoxSubsamplers_for_fraction(fraction, maxTries=maxTries) methods += boxCreator.get_randomQuarterBoxSubsamplers_for_fraction(fraction, maxTries=maxTries)
methods.append(cmeth.TrainedSubsampling(particleContainer, fraction)) # methods.append(cmeth.TrainedSubsampling(particleContainer, fraction))
# methods.append(cmeth.ChemometricSubsampling(particleContainer, fraction)) # methods.append(cmeth.ChemometricSubsampling(particleContainer, fraction))
return methods return methods
...@@ -71,6 +72,14 @@ def is_MP_particle(particle: Particle) -> bool: ...@@ -71,6 +72,14 @@ def is_MP_particle(particle: Particle) -> bool:
return isMP return isMP
def get_number_of_MP_particles(particleList: list) -> int:
numMPParticles = 0
for particle in particleList:
if is_MP_particle(particle):
numMPParticles += 1
return numMPParticles
class TotalResults(object): class TotalResults(object):
def __init__(self): def __init__(self):
super(TotalResults, self).__init__() super(TotalResults, self).__init__()
...@@ -102,11 +111,12 @@ class TotalResults(object): ...@@ -102,11 +111,12 @@ class TotalResults(object):
if multiprocessing: if multiprocessing:
forceList: list = [force]*len(self.sampleResults) forceList: list = [force]*len(self.sampleResults)
indices: list = list(np.arange(len(self.sampleResults))) indices: list = list(np.arange(len(self.sampleResults)))
numSamples: int = len(forceList) # numSamples: int = len(forceList)
numWorkers: int = 4 # in case of quadcore processor that seams reasonable?? # numWorkers: int = 4 # in case of quadcore processor that seams reasonable??
chunksize: int = int(round(numSamples / numWorkers * 0.7)) # we want to have slightly more chunks than workers # chunksize: int = int(round(numSamples / numWorkers * 0.7)) # we want to have slightly more chunks than workers
print(f'multiprocessing with {numSamples} samples and chunksize of {chunksize}') # print(f'multiprocessing with {numSamples} samples and chunksize of {chunksize}')
chunksize = 1
with concurrent.futures.ProcessPoolExecutor() as executor: with concurrent.futures.ProcessPoolExecutor() as executor:
results = executor.map(update_sample, self.sampleResults, forceList, indices, chunksize=chunksize) results = executor.map(update_sample, self.sampleResults, forceList, indices, chunksize=chunksize)
...@@ -129,9 +139,11 @@ class TotalResults(object): ...@@ -129,9 +139,11 @@ class TotalResults(object):
Value: {Dict: Key:Measured Fraction, Value: Tuple (averaged MPCountError, StDev MPCountError) over all samples} Value: {Dict: Key:Measured Fraction, Value: Tuple (averaged MPCountError, StDev MPCountError) over all samples}
""" """
result: dict = {} result: dict = {}
numSamples: int = 0
for sample in self.sampleResults: for sample in self.sampleResults:
sample: SampleResult = sample sample: SampleResult = sample
if attributes == [] or sample.has_any_attribute(attributes): if attributes == [] or sample.has_any_attribute(attributes):
numSamples += 1
for res in sample.results: for res in sample.results:
res: SubsamplingResult = res res: SubsamplingResult = res
method: meth.SubsamplingMethod = res.method method: meth.SubsamplingMethod = res.method
...@@ -155,7 +167,7 @@ class TotalResults(object): ...@@ -155,7 +167,7 @@ class TotalResults(object):
meanStd = np.mean([i[1] for i in methodRes[fraction]]) meanStd = np.mean([i[1] for i in methodRes[fraction]])
methodRes[fraction] = (meanError, meanStd) methodRes[fraction] = (meanError, meanStd)
return result return numSamples, result
class SubsamplingResult(object): class SubsamplingResult(object):
...@@ -227,9 +239,9 @@ class SubsamplingResult(object): ...@@ -227,9 +239,9 @@ class SubsamplingResult(object):
return binSorter.bins, mpCountErrorsPerBin return binSorter.bins, mpCountErrorsPerBin
def _get_mp_count_error(self, allParticles: list, subParticles: list, fractionMeasured: float) -> float: def _get_mp_count_error(self, allParticles: list, subParticles: list, fractionMeasured: float) -> float:
numMPOrig = self._get_number_of_MP_particles(allParticles) numMPOrig = get_number_of_MP_particles(allParticles)
self.origMPCount = numMPOrig self.origMPCount = numMPOrig
numMPEstimate = self._get_number_of_MP_particles(subParticles) / fractionMeasured numMPEstimate = get_number_of_MP_particles(subParticles) / fractionMeasured
self.estimMPCounts.append(numMPEstimate) self.estimMPCounts.append(numMPEstimate)
if numMPOrig != 0: if numMPOrig != 0:
...@@ -245,19 +257,12 @@ class SubsamplingResult(object): ...@@ -245,19 +257,12 @@ class SubsamplingResult(object):
assert (exact != 0) assert (exact != 0)
return abs(exact - estimate) / exact * 100 return abs(exact - estimate) / exact * 100
def _get_number_of_MP_particles(self, particleList: list) -> int:
numMPParticles = 0
for particle in particleList:
if is_MP_particle(particle):
numMPParticles += 1
return numMPParticles
class SampleResult(object): class SampleResult(object):
""" """
An object the stores all generated results per sample and can update and report on them. An object the stores all generated results per sample and can update and report on them.
""" """
def __init__(self, filepath: str, numVariations: int = 5): def __init__(self, filepath: str, numVariations: int = 10):
super(SampleResult, self).__init__() super(SampleResult, self).__init__()
self.filepath: str = filepath self.filepath: str = filepath
self.dataset: dataset.DataSet = None self.dataset: dataset.DataSet = None
...@@ -332,8 +337,38 @@ class SampleResult(object): ...@@ -332,8 +337,38 @@ class SampleResult(object):
return hasAttr return hasAttr
def has_attribute(self, attribute: str) -> bool: def has_attribute(self, attribute: str) -> bool:
attributes: list = [attr.lower() for attr in self.attributes] hasAttr: bool = False
return attribute.lower() in attributes if attribute.find('$') == -1:
attributes: list = [attr.lower() for attr in self.attributes]
hasAttr = attribute.lower() in attributes
else:
operators: dict = {
'<': operator.__lt__,
'<=': operator.__le__,
'==': operator.__eq__,
'!=': operator.__ne__,
'>=': operator.__ge__,
'>': operator.__gt__
}
if self.dataset is None:
self.load_dataset()
valueToCompare: float = 0.0
dsetValue: float = 0.0
for string in operators.keys():
if attribute.find(string) != -1:
valueToCompare = float(attribute.split(string)[1])
if attribute.find('numParticles') != -1:
dsetValue = len(self.dataset.particleContainer.particles)
break
elif attribute.find('mpFraction') != -1:
particles: list = self.dataset.particleContainer.particles
dsetValue = get_number_of_MP_particles(particles) / len(particles)
break
hasAttr = operators[string](dsetValue, valueToCompare)
return hasAttr
def _remove_result_of_method(self, method: meth.SubsamplingMethod) -> None: def _remove_result_of_method(self, method: meth.SubsamplingMethod) -> None:
""" """
......
...@@ -21,7 +21,7 @@ def box_overlaps_other_box(topLeft1: list, topLeft2: list, boxSize: float) -> bo ...@@ -21,7 +21,7 @@ def box_overlaps_other_box(topLeft1: list, topLeft2: list, boxSize: float) -> bo
class BoxSelectionSubsamplingMethod(SubsamplingMethod): class BoxSelectionSubsamplingMethod(SubsamplingMethod):
possibleBoxNumbers: list = [7, 10, 15] possibleBoxNumbers: list = [5, 10, 20]
def __init__(self, *args): def __init__(self, *args):
super(BoxSelectionSubsamplingMethod, self).__init__(*args) super(BoxSelectionSubsamplingMethod, self).__init__(*args)
......
from matplotlib.figure import Figure from matplotlib.figure import Figure
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import numpy as np import numpy as np
from evaluation import TotalResults from evaluation import TotalResults, SampleResult
from helpers import get_filterDimensions_from_dataset, get_center_from_filter_dimensions, convert_length_to_pixels
def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], methods: list = [], def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], methods: list = [],
...@@ -26,8 +27,7 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me ...@@ -26,8 +27,7 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me
index = 0 index = 0
for attrs, meths in zip(attributes, methods): for attrs, meths in zip(attributes, methods):
ax = fig.add_subplot(numRows, numCols, index + 1) ax = fig.add_subplot(numRows, numCols, index + 1)
errorPerFraction: dict = totalResults.get_error_vs_fraction_data(attributes=attrs, numSamples, errorPerFraction = totalResults.get_error_vs_fraction_data(attributes=attrs, methods=meths)
methods=meths)
for methodLabel in errorPerFraction.keys(): for methodLabel in errorPerFraction.keys():
errorDict: dict = errorPerFraction[methodLabel] errorDict: dict = errorPerFraction[methodLabel]
...@@ -45,12 +45,15 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me ...@@ -45,12 +45,15 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me
title: str = '' title: str = ''
if len(attrs) > 0: if len(attrs) > 0:
title = ', '.join(attr for attr in attrs) title = ', '.join(attr for attr in attrs)
print('title is', title) title += f' ({numSamples} samples)'
ax.set_title(title, fontSize=15) ax.set_title(title, fontSize=15)
ax.set_xscale('log') ax.set_xscale('log')
ax.set_xlabel('measured fraction', fontsize=12) ax.set_xlabel('measured fraction', fontsize=12)
ax.set_ylabel('mpCountError (%)', fontsize=12) ax.set_ylabel('mpCountError (%)', fontsize=12)
ax.set_xlim([0.9 * min(fractions), 1.05]) minX, maxX = 0.9 * min(fractions), 1.05
ax.hlines([20, 40, 60, 80], minX, maxX, colors='gray', alpha=0.5)
ax.set_xlim([minX, maxX])
ax.set_ylim([0, 100]) ax.set_ylim([0, 100])
ax.legend() ax.legend()
...@@ -61,20 +64,74 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me ...@@ -61,20 +64,74 @@ def get_error_vs_frac_plot(totalResults: TotalResults, attributes: list = [], me
return fig return fig
def get_distance_hist_plots(totalResults: TotalResults, attributes: list = []) -> Figure:
fig: Figure = plt.figure(figsize=(10, 5))
numRows: int = 1
numCols: int = 1
if len(attributes) == 0:
attributes = [[]]
elif len(attributes) <= 2:
numCols = len(attributes)
else:
numRows = 2
numCols = np.ceil(len(attributes) / numRows)
for index, attrs in enumerate(attributes):
ax = fig.add_subplot(numRows, numCols, index + 1)
densities: list = []
for sampleRes in totalResults.sampleResults:
if sampleRes.has_any_attribute(attrs):
if sampleRes.dataset is None:
sampleRes.load_dataset()
dset = sampleRes.dataset
offset, diameter, [width, height] = get_filterDimensions_from_dataset(dset)
center = get_center_from_filter_dimensions(offset, diameter)
center[0] = convert_length_to_pixels(dset, center[0])
center[1] = convert_length_to_pixels(dset, center[1])
histdata = get_distance_point_histogramdata(dset.particleContainer.particles, center)
densities.append(histdata[1])
# ax.plot(histdata[0], histdata[1])
numSamples = len(densities)
title: str = ''
if len(attrs) > 0:
title = ', '.join(attr for attr in attrs)
title += f' ({numSamples} samples)'
ax.set_title(title, fontSize=15)
densities: np.ndarray = np.mean(np.array(densities), axis=0)
ax.plot(histdata[0], densities)
return fig
def get_distance_point_histogramdata(particles: list, center: np.ndarray) -> tuple:
"""
:param particles: list of Particles
:param center: np.array([x, y]) of center point, in px
:return histogramdata: tuple: (center bin dist , particle count)
"""
def get_area_of_circle_ring(innerRadius: float, outerRadius: float) -> float:
area: float = np.pi * (outerRadius + innerRadius) * (outerRadius - innerRadius)
return area
maxUpperLimit: float = 1E4
numBins: int = 11
bins: np.ndarray = np.linspace(0, maxUpperLimit, numBins, endpoint=True)
particleCenters: list = []
for particle in particles:
particleCenters.append([np.mean(particle.contour[:, 0, 0]), np.mean(particle.contour[:, 0, 1])])
distancesToPoints: np.ndarray = np.linalg.norm(particleCenters - center, axis=1)
data, binMaxima = np.histogram(distancesToPoints, bins)
densities: np.ndarray = np.zeros_like(data, dtype=np.float)
for i in range(len(data)):
densities[i] = float(data[i]) / get_area_of_circle_ring(binMaxima[i], binMaxima[i+1])
densities /= densities.max()
binCenters: list = [np.mean([binMaxima[i], binMaxima[i+1]]) for i in range(len(binMaxima)-1)]
return binCenters, densities
# def get_grouped_spectra_plot(groupedSpectra: list, wavenumbers=None) -> matplotlib.figure.Figure:
# if wavenumbers is None:
# wavenumbers = np.arange(len(groupedSpectra[0][0]))
#
# numLabels = len(groupedSpectra)
# numRows = numLabels // 3
# numCols = np.ceil(numLabels / numRows)
# fig: matplotlib.figure.Figure = plt.figure()
#
# for index, specs in enumerate(groupedSpectra):
# ax = fig.add_subplot(numRows, numCols, index + 1)
# for spec in specs:
# ax.plot(wavenumbers, spec)
# ax.set_title(f'{len(specs)} spectra of label {index + 1}')
#
# return fig
\ No newline at end of file
...@@ -4,7 +4,7 @@ import time ...@@ -4,7 +4,7 @@ import time
from evaluation import TotalResults, SampleResult from evaluation import TotalResults, SampleResult
from input_output import get_pkls_from_directory, get_attributes_from_foldername, save_results, load_results from input_output import get_pkls_from_directory, get_attributes_from_foldername, save_results, load_results
from graphs import get_error_vs_frac_plot from graphs import get_error_vs_frac_plot, get_distance_hist_plots
""" """
IMPORTANT!!! IMPORTANT!!!
...@@ -12,29 +12,33 @@ SET GEPARD TO EVALUATION BRANCH (WITHOUT THE TILING STUFF), OTHERWISE SOME OF TH ...@@ -12,29 +12,33 @@ SET GEPARD TO EVALUATION BRANCH (WITHOUT THE TILING STUFF), OTHERWISE SOME OF TH
""" """
if __name__ == '__main__': if __name__ == '__main__':
results: TotalResults = TotalResults() # results: TotalResults = TotalResults()
pklsInFolders = get_pkls_from_directory(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets') # pklsInFolders = get_pkls_from_directory(r'C:\Users\xbrjos\Desktop\temp MP\NewDatasets')
counter = 0 # counter = 0
for folder in pklsInFolders.keys(): # for folder in pklsInFolders.keys():
for samplePath in pklsInFolders[folder]: # for samplePath in pklsInFolders[folder]:
if counter < 10: # if counter < 100:
newSampleResult: SampleResult = results.add_sample(samplePath) # newSampleResult: SampleResult = results.add_sample(samplePath)
for attr in get_attributes_from_foldername(folder): # for attr in get_attributes_from_foldername(folder):
newSampleResult.set_attribute(attr) # newSampleResult.set_attribute(attr)
counter += 1 # counter += 1
#
# t0 = time.time()
# results.update_all(multiprocessing=True)
# print('updating all took', time.time()-t0, 'seconds')
#
# save_results('results2.res', results)
results: TotalResults = load_results('results2.res')
#
plot: Figure = get_error_vs_frac_plot(results,
attributes=[['air'], ['water'], ['sediment', 'soil', 'beach'], ['slush']],
# # attributes=[['$ mpFraction < 0.02'], ['$ mpFraction >= 0.02']],
# # methods=[['random layout (7', 'random layout (1']]*2)
methods=[['random sub', 'cross', 'layout (10']]*4)
# # methods=[['Random Subsampling', 'Sizebin']] * 2)
# # methods=[['layout (5', 'layout (10', 'layout (15', 'cross', 'random subsampling', 'sizebin']] * 2)
# plot.show()
t0 = time.time() # plot2: Figure = get_distance_hist_plots(results,
results.update_all(multiprocessing=False) # attributes=[['air'], ['water'], ['sediment', 'soil', 'beach'], ['slush']])
print('updating all took', time.time()-t0, 'seconds') # plot2.show()
save_results('results_test.res', results)
# results: TotalResults = load_results('results2.res')
plot: Figure = get_error_vs_frac_plot(results, attributes=[[]],
methods=[['random', 'trained']], standarddevs=True)
# plot: Figure = get_error_vs_frac_plot(results, attributes=[['air', 'water'], ['sediment', 'soil', 'beach', 'slush']],
# methods=[['random layout (7', 'random layout (1']]*2)
# methods=[[]]*2)
# methods=[['Random Subsampling', 'Sizebin']] * 2)
# methods=[['layout (7', 'layout (10', 'layout (15', 'cross', 'random subsampling', 'sizebin']] * 2)
plot.show()
...@@ -21,8 +21,33 @@ from evaluation import SubsamplingResult ...@@ -21,8 +21,33 @@ from evaluation import SubsamplingResult
class TestParticleFeatures(unittest.TestCase): class TestParticleFeatures(unittest.TestCase):
def test_image_features(self):
img: np.ndarray = np.zeros((100, 100, 3), dtype=np.uint8)
img[:, :, 0] = 255 # we just have a plain red image
imgFeatureVector: np.ndarray = cmeth.get_image_feature_vec(img)
self.assertEqual(imgFeatureVector.shape[0], 6)
def test_get_mean_color_and_stdev(self):
img: np.ndarray = np.zeros((100, 100, 3), dtype=np.uint8)
img[:, :, 0] = 255 # we just have a plain red image
meanStd = cmeth.get_mean_and_stdev(img)
self.assertTrue(np.array_equal(meanStd[0], np.array([255, 0, 0])))
self.assertTrue(np.array_equal(meanStd[1], np.array([0, 0, 0])))
img[:, :, 1] = 255
meanStd = cmeth.get_mean_and_stdev(img)
self.assertTrue(np.array_equal(meanStd[0], np.array([255, 255, 0])))
self.assertTrue(np.array_equal(meanStd[1], np.array([0, 0, 0])))
img[:50, :50, 0] = 128
meanStd = cmeth.get_mean_and_stdev(img)
meanRed: float = np.mean([128, 255, 255, 255])
stdRed: float = np.std([128, 255, 255, 255])
self.assertTrue(np.array_equal(meanStd[0], np.array([meanRed, 255, 0])))
self.assertTrue(np.array_equal(meanStd[1], np.array([stdRed, 0, 0])))
def test_get_contour_moments(self): def test_get_contour_moments(self):
imgs = [] imgs: list = []
imgA: np.ndarray = np.zeros((200, 200), dtype=np.uint8) imgA: np.ndarray = np.zeros((200, 200), dtype=np.uint8)
cv2.putText(imgA, 'A', (25, 175), fontFace=cv2.FONT_HERSHEY_SIMPLEX, cv2.putText(imgA, 'A', (25, 175), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=7, color=1, thickness=5) fontScale=7, color=1, thickness=5)
...@@ -51,6 +76,25 @@ class TestParticleFeatures(unittest.TestCase): ...@@ -51,6 +76,25 @@ class TestParticleFeatures(unittest.TestCase):
diff: np.ndarray = moments[i, :] - np.mean(moments[i, :]) diff: np.ndarray = moments[i, :] - np.mean(moments[i, :])
self.assertFalse(np.any(diff > 0.1)) self.assertFalse(np.any(diff > 0.1))
def test_get_curvature_ft(self):
def get_cnt_from_img(binImg: np.ndarray) -> np.ndarray:
contours, hierarchy = cv2.findContours(binImg, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)
return contours[0]
harmonics: list = []
for shape in [cv2.MORPH_CROSS, cv2.MORPH_ELLIPSE, cv2.MORPH_RECT]:
for size in [50, 500]:
padding = round(size/10)
img: np.ndarray = np.zeros((size + 2*padding, size + 2*padding))
img[padding:size+padding, padding:size+padding] = cv2.getStructuringElement(shape, (size, size))
cnt: np.ndarray = get_cnt_from_img(img.astype(np.uint8))
for numHarmonics in [2, 5, 15]:
dft: np.ndarray = cmeth.get_curvature_ft(cnt, numHarmonics=numHarmonics)
harmonics.append(dft)
self.assertEqual(dft.shape[0], numHarmonics)
def test_get_color_hash(self): def test_get_color_hash(self):
for color in ['red', 'green', 'violet', 'blue', 'Blue', 'non-determinable', None]: for color in ['red', 'green', 'violet', 'blue', 'Blue', 'non-determinable', None]:
for numDigits in [4, 6, 8]: for numDigits in [4, 6, 8]:
...@@ -70,6 +114,8 @@ class TestParticleFeatures(unittest.TestCase): ...@@ -70,6 +114,8 @@ class TestParticleFeatures(unittest.TestCase):
particleContainer: ParticleContainer = get_default_ParticleContainer() particleContainer: ParticleContainer = get_default_ParticleContainer()
features: np.ndarray = cmeth.get_particle_featurematrix(particleContainer) features: np.ndarray = cmeth.get_particle_featurematrix(particleContainer)
self.assertEqual(features.shape[0], len(particleContainer.particles)) self.assertEqual(features.shape[0], len(particleContainer.particles))
for entry in features[0, :]:
self.assertTrue(type(entry) in [float, np.float64])
class TestTrainedSubsampling(unittest.TestCase): class TestTrainedSubsampling(unittest.TestCase):
......
...@@ -10,12 +10,12 @@ import unittest ...@@ -10,12 +10,12 @@ import unittest
import sys import sys
sys.path.append("C://Users//xbrjos//Desktop//Python") sys.path.append("C://Users//xbrjos//Desktop//Python")
import gepard import gepard
from gepard.dataset import DataSet
from gepard.analysis.particleContainer import ParticleContainer from gepard.analysis.particleContainer import ParticleContainer
from evaluation import TotalResults, SampleResult, SubsamplingResult, get_methods_to_test from evaluation import TotalResults, SampleResult, SubsamplingResult, get_methods_to_test, get_number_of_MP_particles
import methods as meth import methods as meth
import geometricMethods as gmeth import geometricMethods as gmeth
from chemometrics.chemometricMethods import TrainedSubsampling
from helpers_for_test import get_default_ParticleContainer, get_default_DataSet, get_MP_particles, get_non_MP_particles, get_MP_particle, get_non_MP_particle from helpers_for_test import get_default_ParticleContainer, get_default_DataSet, get_MP_particles, get_non_MP_particles, get_MP_particle, get_non_MP_particle
...@@ -76,7 +76,8 @@ class TestTotalResults(unittest.TestCase): ...@@ -76,7 +76,8 @@ class TestTotalResults(unittest.TestCase):
firstSample.results = [firstResult, secondResult, thirdResult, thirdResult3] firstSample.results = [firstResult, secondResult, thirdResult, thirdResult3]
secondSample.results = [firstResult, secondResult, thirdResult2, thirdResult3] secondSample.results = [firstResult, secondResult, thirdResult2, thirdResult3]
resultDict: dict = self.totalResults.get_error_vs_fraction_data() numSamples, resultDict = self.totalResults.get_error_vs_fraction_data()
self.assertEqual(numSamples, 2)
self.assertEqual(list(resultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label]) self.assertEqual(list(resultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
for i, key in enumerate(resultDict.keys()): for i, key in enumerate(resultDict.keys()):
...@@ -100,7 +101,8 @@ class TestTotalResults(unittest.TestCase): ...@@ -100,7 +101,8 @@ class TestTotalResults(unittest.TestCase):
self.assertAlmostEqual(mean, 50) self.assertAlmostEqual(mean, 50)
self.assertAlmostEqual(stdev, 16.32993161855452) self.assertAlmostEqual(stdev, 16.32993161855452)
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(attributes=['to be used']) numSamples, filteredResultDict = self.totalResults.get_error_vs_fraction_data(attributes=['to be used'])
self.assertEqual(numSamples, 1)
self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label]) self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label, secondMethod.label, thirdMethod.label])
for i, key in enumerate(filteredResultDict.keys()): for i, key in enumerate(filteredResultDict.keys()):
...@@ -127,13 +129,16 @@ class TestTotalResults(unittest.TestCase): ...@@ -127,13 +129,16 @@ class TestTotalResults(unittest.TestCase):
self.assertAlmostEqual(mean, 50) self.assertAlmostEqual(mean, 50)
self.assertAlmostEqual(stdev, 16.32993161855452) self.assertAlmostEqual(stdev, 16.32993161855452)
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['cross']) numSamples, filteredResultDict = self.totalResults.get_error_vs_fraction_data(methods=['cross'])
self.assertEqual(numSamples, 2)
self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label]) self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['Cross']) numSamples, filteredResultDict = self.totalResults.get_error_vs_fraction_data(methods=['Cross'])
self.assertEqual(numSamples, 2)
self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label]) self.assertEqual(list(filteredResultDict.keys()), [secondMethod.label, thirdMethod.label])
filteredResultDict: dict = self.totalResults.get_error_vs_fraction_data(methods=['random']) numSamples, filteredResultDict = self.totalResults.get_error_vs_fraction_data(methods=['random'])
self.assertEqual(numSamples, 2)
self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label]) self.assertEqual(list(filteredResultDict.keys()), [firstMethod.label])
...@@ -233,6 +238,22 @@ class TestSampleResult(unittest.TestCase): ...@@ -233,6 +238,22 @@ class TestSampleResult(unittest.TestCase):
self.assertFalse(self.sampleResult.has_any_attribute(['water', 'sediment'])) self.assertFalse(self.sampleResult.has_any_attribute(['water', 'sediment']))
self.assertFalse(self.sampleResult.has_any_attribute(['beach'])) self.assertFalse(self.sampleResult.has_any_attribute(['beach']))
# keywords with $ allow filtering for properties from the dataset itself
self.sampleResult.dataset = get_default_DataSet()
self.sampleResult.dataset.particleContainer.initializeParticles(999)
self.assertTrue(self.sampleResult.has_attribute('$ numParticles < 1000'))
self.assertFalse(self.sampleResult.has_attribute('$ numParticles >= 1000'))
self.sampleResult.dataset.particleContainer.initializeParticles(1000)
self.assertFalse(self.sampleResult.has_attribute('$ numParticles < 1000'))
self.assertTrue(self.sampleResult.has_attribute('$ numParticles >= 1000'))
fifty_percent_mp: list = get_non_MP_particles(100) + get_MP_particles(100)
self.sampleResult.dataset.particleContainer.particles = fifty_percent_mp
self.assertTrue(self.sampleResult.has_attribute('$ mpFraction == 0.5'))
self.assertFalse(self.sampleResult.has_attribute('$ mpFraction > 0.5'))
self.assertFalse(self.sampleResult.has_attribute('$ mpFraction < 0.5'))
def test_get_methods_to_test(self): def test_get_methods_to_test(self):
def containsMethod(listOfMethods: list, template: meth.SubsamplingMethod) -> bool: def containsMethod(listOfMethods: list, template: meth.SubsamplingMethod) -> bool:
contains: bool = False contains: bool = False
...@@ -258,7 +279,7 @@ class TestSampleResult(unittest.TestCase): ...@@ -258,7 +279,7 @@ class TestSampleResult(unittest.TestCase):
possibleSpiralBoxMethods = 3 possibleSpiralBoxMethods = 3
possibleRandomBoxMethods = 3 possibleRandomBoxMethods = 3
possibleQuarterRandomBoxMethods = 3 possibleQuarterRandomBoxMethods = 3
possibleChemometricMethods = 1 possibleChemometricMethods = 0
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \ totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \
possibleSpiralBoxMethods + possibleChemometricMethods + \ possibleSpiralBoxMethods + possibleChemometricMethods + \
possibleRandomBoxMethods + possibleQuarterRandomBoxMethods possibleRandomBoxMethods + possibleQuarterRandomBoxMethods
...@@ -277,7 +298,7 @@ class TestSampleResult(unittest.TestCase): ...@@ -277,7 +298,7 @@ class TestSampleResult(unittest.TestCase):
possibleSpiralBoxMethods = 0 possibleSpiralBoxMethods = 0
possibleRandomBoxMethods = 0 possibleRandomBoxMethods = 0
possibleQuarterRandomBoxMethods = 0 possibleQuarterRandomBoxMethods = 0
possibleChemometricMethods = 1 possibleChemometricMethods = 0
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \ totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \
possibleSpiralBoxMethods + possibleChemometricMethods + \ possibleSpiralBoxMethods + possibleChemometricMethods + \
possibleRandomBoxMethods + possibleQuarterRandomBoxMethods possibleRandomBoxMethods + possibleQuarterRandomBoxMethods
...@@ -294,7 +315,7 @@ class TestSampleResult(unittest.TestCase): ...@@ -294,7 +315,7 @@ class TestSampleResult(unittest.TestCase):
possibleSpiralBoxMethods = 0 possibleSpiralBoxMethods = 0
possibleRandomBoxMethods = 0 possibleRandomBoxMethods = 0
possibleQuarterRandomBoxMethods = 0 possibleQuarterRandomBoxMethods = 0
possibleChemometricMethods = 1 possibleChemometricMethods = 0
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \ totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \
possibleSpiralBoxMethods + possibleChemometricMethods + \ possibleSpiralBoxMethods + possibleChemometricMethods + \
possibleRandomBoxMethods + possibleQuarterRandomBoxMethods possibleRandomBoxMethods + possibleQuarterRandomBoxMethods
...@@ -309,7 +330,7 @@ class TestSampleResult(unittest.TestCase): ...@@ -309,7 +330,7 @@ class TestSampleResult(unittest.TestCase):
possibleRandomMethods = 4 possibleRandomMethods = 4
possibleCrossBoxMethods = 3 possibleCrossBoxMethods = 3
possibleSpiralBoxMethods = 3 possibleSpiralBoxMethods = 3
possibleChemometricMethods = 2 possibleChemometricMethods = 0
possibleRandomBoxMethods = 3 possibleRandomBoxMethods = 3
possibleQuarterRandomBoxMethods = 3 possibleQuarterRandomBoxMethods = 3
totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \ totalPossible = possibleCrossBoxMethods + possibleRandomMethods + \
...@@ -456,7 +477,7 @@ class TestSubsamplingResult(unittest.TestCase): ...@@ -456,7 +477,7 @@ class TestSubsamplingResult(unittest.TestCase):
allParticles = mpParticles + nonMPparticles allParticles = mpParticles + nonMPparticles
calculatedNumMPParticles = self.subsamplingResult._get_number_of_MP_particles(allParticles) calculatedNumMPParticles = get_number_of_MP_particles(allParticles)
self.assertEqual(numMPParticles, calculatedNumMPParticles) self.assertEqual(numMPParticles, calculatedNumMPParticles)
def test_get_mp_count_error(self): def test_get_mp_count_error(self):
......