Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Open sidebar
Josef Brandt
Subsampling
Commits
229dd4ac
Commit
229dd4ac
authored
Mar 13, 2020
by
Josef Brandt
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Basic concept of result objects
parent
fa14c5fa
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
415 additions
and
233 deletions
+415
-233
evaluation.py
evaluation.py
+138
-10
geometricMethods.py
geometricMethods.py
+23
-22
gui/measureModes.py
gui/measureModes.py
+3
-3
helpers.py
helpers.py
+18
-0
methods.py
methods.py
+0
-25
subsampling.py
subsampling.py
+8
-67
tests/test_evaluation.py
tests/test_evaluation.py
+143
-26
tests/test_geometricMethods.py
tests/test_geometricMethods.py
+62
-53
tests/test_helpers.py
tests/test_helpers.py
+19
-0
tests/test_methods.py
tests/test_methods.py
+1
-27
No files found.
evaluation.py
View file @
229dd4ac
...
@@ -6,30 +6,158 @@ Created on Wed Jan 22 13:57:28 2020
...
@@ -6,30 +6,158 @@ Created on Wed Jan 22 13:57:28 2020
@author: luna
@author: luna
"""
"""
import
pickle
import
pickle
import
sys
import
os
import
os
from
helpers
import
ParticleBinSorter
from
helpers
import
ParticleBinSorter
import
methods
as
meth
import
geometricMethods
as
gmeth
sys
.
path
.
append
(
"C://Users//xbrjos//Desktop//Python"
)
from
gepard
import
dataset
class
ResultObject
(
object
):
def
get_name_from_directory
(
dirPath
:
str
)
->
str
:
return
str
(
os
.
path
.
basename
(
dirPath
).
split
(
'.'
)[
0
])
class
TotalResults
(
object
):
methods
:
list
=
[
meth
.
RandomSampling
,
meth
.
SizeBinFractioning
,
gmeth
.
CrossBoxSubSampling
,
gmeth
.
SpiralBoxSubsampling
]
measuredFreactions
:
list
=
[
0.1
,
0.3
,
0.5
,
0.9
]
def
__init__
(
self
):
super
(
TotalResults
,
self
).
__init__
()
self
.
sampleResults
:
list
=
[]
def
add_sample
(
self
,
filePath
:
str
)
->
None
:
"""
Adds a new sampleResult object, if a .pkl file is given and if the sample name is not already present.
:param filePath:
:return:
"""
sampleName
:
str
=
get_name_from_directory
(
filePath
)
presentSampleNames
:
list
=
[
res
.
sampleName
for
res
in
self
.
sampleResults
]
if
sampleName
not
in
presentSampleNames
:
if
os
.
path
.
basename
(
filePath
).
split
(
'.'
)[
-
1
]
==
'pkl'
:
self
.
sampleResults
.
append
(
SampleResult
(
filePath
))
def
update_all
(
self
)
->
None
:
"""
Updates all samples with all methods and all fractions
:return:
"""
for
sample
in
self
.
sampleResults
:
sample
.
load_dataset
()
for
fraction
in
self
.
measuredFreactions
:
possibleMethods
=
self
.
_get_methods_for_fraction
(
sample
.
dataset
,
fraction
)
for
curMethod
in
possibleMethods
:
print
(
f
'updating
{
sample
.
sampleName
}
with
{
curMethod
.
label
}
at fraction
{
fraction
}
'
)
sample
.
update_result_with_method
(
curMethod
)
def
_get_methods_for_fraction
(
self
,
dataset
:
dataset
.
DataSet
,
fraction
:
float
)
->
list
:
"""
:param fraction: The desired fraction to measure
:return: list of measurement Objects that are applicable
"""
particleContainer
=
dataset
.
particleContainer
methods
:
list
=
[
meth
.
RandomSampling
(
particleContainer
,
fraction
),
meth
.
SizeBinFractioning
(
particleContainer
,
fraction
)]
boxCreator
:
gmeth
.
BoxSelectionCreator
=
gmeth
.
BoxSelectionCreator
(
dataset
)
methods
+=
boxCreator
.
get_crossBoxSubsamplers_for_fraction
(
fraction
)
methods
+=
boxCreator
.
get_spiralBoxSubsamplers_for_fraction
(
fraction
)
return
methods
class
SampleResult
(
object
):
"""
"""
An object the actually stores all generated results per sample and can update and report on them.
An object the actually stores all generated results per sample and can update and report on them.
"""
"""
def
__init__
(
self
,
filepath
:
str
):
def
__init__
(
self
,
filepath
:
str
):
super
(
ResultObjec
t
,
self
).
__init__
()
super
(
SampleResul
t
,
self
).
__init__
()
self
.
filepath
:
str
=
filepath
self
.
filepath
:
str
=
filepath
print
(
self
.
sampleName
)
self
.
dataset
:
dataset
.
DataSet
=
None
self
.
results
:
list
=
[]
# noinspection PyTypeChecker
@
property
@
property
def
sampleName
(
self
)
->
str
:
def
sampleName
(
self
)
->
str
:
return
os
.
path
.
basename
(
self
.
filepath
)
.
split
(
'.'
)[
0
]
return
get_name_from_directory
(
self
.
filepath
)
def
load_dataset
(
self
)
->
None
:
self
.
dataset
=
dataset
.
loadData
(
self
.
filepath
)
def
update_result_with_method
(
self
,
method
:
meth
.
SubsamplingMethod
,
force
:
bool
=
False
)
->
None
:
"""
Updates result with the given method (contains desiredFraction already)
:param method: The SubsamplingMethod Object
:param force: Wether to force an update. If False, the result is not updated, if it is already present.
:return:
"""
if
not
self
.
_result_is_already_present
(
method
)
or
force
:
if
force
:
self
.
_remove_result_of_method
(
method
)
if
self
.
dataset
is
None
:
self
.
load_dataset
()
method
.
particleContainer
=
self
.
dataset
.
particleContainer
newResult
:
SubsamplingResult
=
SubsamplingResult
(
method
)
self
.
results
.
append
(
newResult
)
newResult
.
update
()
def
_remove_result_of_method
(
self
,
method
:
meth
.
SubsamplingMethod
)
->
None
:
"""
Removes the specified result from the list
:param method:
:return:
"""
for
result
in
self
.
results
:
if
type
(
result
.
method
)
==
type
(
method
)
and
result
.
fraction
==
method
.
fraction
:
self
.
results
.
remove
(
result
)
def
_result_is_already_present
(
self
,
method
:
meth
.
SubsamplingMethod
)
->
bool
:
"""
Checks, if a result with the given method (method type AND measured fraction) is already present.
:param method: The method object, specifying the subsampling method and the measured fraction
:return:
"""
isPresent
:
bool
=
False
for
result
in
self
.
results
:
if
type
(
result
.
method
)
==
type
(
method
)
and
result
.
fraction
==
method
.
fraction
:
isPresent
=
True
break
return
isPresent
class
SubsamplingResult
(
object
):
"""
Stores all interesting results from a subsampling experiment
"""
def
__init__
(
self
,
subsamplingMethod
:
meth
.
SubsamplingMethod
):
super
(
SubsamplingResult
,
self
).
__init__
()
self
.
method
:
meth
.
SubsamplingMethod
=
subsamplingMethod
self
.
fraction
=
self
.
method
.
fraction
self
.
origParticleCount
:
int
=
None
self
.
subSampledParticleCount
:
int
=
None
self
.
mpCountError
:
float
=
None
self
.
mpCountErrorPerBin
:
tuple
=
None
def
update
(
self
)
->
None
:
"""
Updates all results from the method.
:return:
"""
assert
self
.
method
.
particleContainer
is
not
None
origParticles
:
list
=
self
.
method
.
particleContainer
.
particles
self
.
origParticleCount
=
len
(
origParticles
)
subParticles
:
list
=
self
.
method
.
apply_subsampling_method
()
self
.
subSampledParticleCount
=
len
(
subParticles
)
fraction
:
float
=
self
.
method
.
fraction
self
.
mpCountError
=
self
.
_get_mp_count_error
(
origParticles
,
subParticles
,
fraction
)
self
.
mpCountErrorPerBin
=
self
.
_get_mp_count_error_per_bin
(
origParticles
,
subParticles
,
fraction
)
class
ResultComparer
(
object
):
def
_get_mp_count_error_per_bin
(
self
,
allParticles
:
list
,
subParticles
:
list
,
fractionMeasured
:
float
)
->
tuple
:
def
_get_mp_count_error_per_bin
(
self
,
allParticles
,
subParticles
,
fractionMeasured
):
binSorter
=
ParticleBinSorter
()
binSorter
=
ParticleBinSorter
()
allParticlesInBins
=
binSorter
.
sort_particles_into_bins
(
allParticles
)
allParticlesInBins
=
binSorter
.
sort_particles_into_bins
(
allParticles
)
subParticlesInBins
=
binSorter
.
sort_particles_into_bins
(
subParticles
)
subParticlesInBins
=
binSorter
.
sort_particles_into_bins
(
subParticles
)
...
@@ -38,7 +166,7 @@ class ResultComparer(object):
...
@@ -38,7 +166,7 @@ class ResultComparer(object):
mpCountErrorsPerBin
.
append
(
self
.
_get_mp_count_error
(
allParticleBin
,
subParticleBin
,
fractionMeasured
))
mpCountErrorsPerBin
.
append
(
self
.
_get_mp_count_error
(
allParticleBin
,
subParticleBin
,
fractionMeasured
))
return
binSorter
.
bins
,
mpCountErrorsPerBin
return
binSorter
.
bins
,
mpCountErrorsPerBin
def
_get_mp_count_error
(
self
,
allParticles
,
subParticles
,
fractionMeasured
)
:
def
_get_mp_count_error
(
self
,
allParticles
:
list
,
subParticles
:
list
,
fractionMeasured
:
float
)
->
float
:
numMPOrig
=
self
.
_get_number_of_MP_particles
(
allParticles
)
numMPOrig
=
self
.
_get_number_of_MP_particles
(
allParticles
)
numMPEstimate
=
self
.
_get_number_of_MP_particles
(
subParticles
)
/
fractionMeasured
numMPEstimate
=
self
.
_get_number_of_MP_particles
(
subParticles
)
/
fractionMeasured
...
@@ -51,11 +179,11 @@ class ResultComparer(object):
...
@@ -51,11 +179,11 @@ class ResultComparer(object):
return
mpCountError
return
mpCountError
def
_get_error_from_values
(
self
,
exact
,
estimate
)
:
def
_get_error_from_values
(
self
,
exact
:
float
,
estimate
:
float
)
->
float
:
assert
(
exact
!=
0
)
assert
(
exact
!=
0
)
return
abs
(
exact
-
estimate
)
/
exact
return
abs
(
exact
-
estimate
)
/
exact
def
_get_number_of_MP_particles
(
self
,
particleList
)
:
def
_get_number_of_MP_particles
(
self
,
particleList
:
list
)
->
int
:
mpPatterns
=
[
'poly'
,
'rubber'
,
'pb'
,
'pr'
,
'pg'
,
'py'
,
'pv'
]
mpPatterns
=
[
'poly'
,
'rubber'
,
'pb'
,
'pr'
,
'pg'
,
'py'
,
'pv'
]
numMPParticles
=
0
numMPParticles
=
0
...
...
geometricMethods.py
View file @
229dd4ac
...
@@ -55,62 +55,63 @@ class BoxSelectionSubsamplingMethod(SubsamplingMethod):
...
@@ -55,62 +55,63 @@ class BoxSelectionSubsamplingMethod(SubsamplingMethod):
newTopLefts
.
append
((
topLeft
[
0
]
+
self
.
offset
[
0
],
topLeft
[
1
]
+
self
.
offset
[
1
]))
newTopLefts
.
append
((
topLeft
[
0
]
+
self
.
offset
[
0
],
topLeft
[
1
]
+
self
.
offset
[
1
]))
return
newTopLefts
return
newTopLefts
class
BoxSelectionCreator
(
object
):
class
BoxSelectionCreator
(
object
):
def
__init__
(
self
,
dataset
:
dataset
.
DataSet
):
def
__init__
(
self
,
dataset
:
dataset
.
DataSet
):
super
(
BoxSelectionCreator
,
self
).
__init__
()
super
(
BoxSelectionCreator
,
self
).
__init__
()
self
.
dataset
:
dataset
.
DataSet
=
dataset
self
.
dataset
:
dataset
.
DataSet
=
dataset
self
.
minNumberOfBoxes
:
int
=
10
self
.
maxNumberOfBoxes
:
int
=
20
def
get_crossBoxS
electo
rs_for_fraction
(
self
,
desiredFraction
:
float
)
->
list
:
def
get_crossBoxS
ubsample
rs_for_fraction
(
self
,
desiredFraction
:
float
)
->
list
:
"""
"""
Creates CrossBoxS
electo
rs that fullfill the desired fraction criterium.
Creates CrossBoxS
ubsample
rs that fullfill the desired fraction criterium.
:param desiredFraction:
:param desiredFraction:
:return list of CrossBoxS
electo
rs:
:return list of CrossBoxS
ubsample
rs:
"""
"""
crossBoxS
electo
rs
=
[]
crossBoxS
ubsample
rs
=
[]
offset
,
diameter
,
widthHeight
=
helpers
.
get_filterDimensions_from_dataset
(
self
.
dataset
)
offset
,
diameter
,
widthHeight
=
helpers
.
get_filterDimensions_from_dataset
(
self
.
dataset
)
diameter
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
diameter
)
diameter
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
diameter
)
offset
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
0
]),
\
offset
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
0
]),
\
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
1
])
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
1
])
for
numBoxesAcross
in
[
3
,
5
]:
for
numBoxesAcross
in
[
3
,
5
]:
newBoxSelector
:
CrossBoxS
elector
=
CrossBoxS
elector
(
self
.
dataset
.
particleContainer
,
desiredFraction
)
newBoxSelector
:
CrossBoxS
ubSampling
=
CrossBoxS
ubSampling
(
self
.
dataset
.
particleContainer
,
desiredFraction
)
newBoxSelector
.
filterDiameter
=
diameter
newBoxSelector
.
filterDiameter
=
diameter
newBoxSelector
.
offset
=
offset
newBoxSelector
.
offset
=
offset
newBoxSelector
.
numBoxesAcross
=
numBoxesAcross
newBoxSelector
.
numBoxesAcross
=
numBoxesAcross
crossBoxSelectors
.
append
(
newBoxSelector
)
maxFraction
:
float
=
newBoxSelector
.
get_maximum_achievable_fraction
()
if
desiredFraction
<=
maxFraction
:
crossBoxSubsamplers
.
append
(
newBoxSelector
)
return
crossBoxS
electo
rs
return
crossBoxS
ubsample
rs
def
get_spiralBoxS
electo
rs_for_fraction
(
self
,
desiredFraction
:
float
)
->
list
:
def
get_spiralBoxS
ubsample
rs_for_fraction
(
self
,
desiredFraction
:
float
)
->
list
:
"""
"""
Creates CrossBoxS
electo
rs that fullfill the desired fraction criterium.
Creates CrossBoxS
ubsample
rs that fullfill the desired fraction criterium.
:param desiredFraction:
:param desiredFraction:
:return list of SpiralBoxSelectors:
:return list of SpiralBoxSelectors:
"""
"""
spiralBoxS
electo
rs
=
[]
spiralBoxS
ubsample
rs
=
[]
offset
,
diameter
,
widthHeight
=
helpers
.
get_filterDimensions_from_dataset
(
self
.
dataset
)
offset
,
diameter
,
widthHeight
=
helpers
.
get_filterDimensions_from_dataset
(
self
.
dataset
)
diameter
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
diameter
)
diameter
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
diameter
)
offset
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
0
]),
\
offset
=
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
0
]),
\
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
1
])
helpers
.
convert_length_to_pixels
(
self
.
dataset
,
offset
[
1
])
for
numBoxes
in
Spiral
Selector
.
possibleBoxNumbers
:
for
numBoxes
in
Spiral
BoxSubsampling
.
possibleBoxNumbers
:
newBoxSelector
:
Spiral
Selector
=
SpiralSelector
(
self
.
dataset
.
particleContainer
,
desiredFraction
)
newBoxSelector
:
Spiral
BoxSubsampling
=
SpiralBoxSubsampling
(
self
.
dataset
.
particleContainer
,
desiredFraction
)
newBoxSelector
.
filterDiameter
=
diameter
newBoxSelector
.
filterDiameter
=
diameter
newBoxSelector
.
offset
=
offset
newBoxSelector
.
offset
=
offset
newBoxSelector
.
numBoxes
=
numBoxes
newBoxSelector
.
numBoxes
=
numBoxes
if
newBoxSelector
.
noBoxOverlap
:
if
newBoxSelector
.
noBoxOverlap
:
spiralBoxS
electo
rs
.
append
(
newBoxSelector
)
spiralBoxS
ubsample
rs
.
append
(
newBoxSelector
)
return
spiralBoxS
electo
rs
return
spiralBoxS
ubsample
rs
class
CrossBoxS
elector
(
BoxSelectionSubsamplingMethod
):
class
CrossBoxS
ubSampling
(
BoxSelectionSubsamplingMethod
):
def
__init__
(
self
,
particleContainer
,
desiredFraction
:
float
=
0.1
)
->
None
:
def
__init__
(
self
,
particleContainer
,
desiredFraction
:
float
=
0.1
)
->
None
:
super
(
CrossBoxS
elector
,
self
).
__init__
(
particleContainer
,
desiredFraction
)
super
(
CrossBoxS
ubSampling
,
self
).
__init__
(
particleContainer
,
desiredFraction
)
self
.
numBoxesAcross
:
int
=
3
# either 3 or 5
self
.
numBoxesAcross
:
int
=
3
# either 3 or 5
@
property
@
property
...
@@ -187,12 +188,12 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
...
@@ -187,12 +188,12 @@ class CrossBoxSelector(BoxSelectionSubsamplingMethod):
return
tileStarts
return
tileStarts
class
Spiral
Selector
(
BoxSelectionSubsamplingMethod
):
class
Spiral
BoxSubsampling
(
BoxSelectionSubsamplingMethod
):
possibleBoxNumbers
:
list
=
[
10
,
15
,
20
]
possibleBoxNumbers
:
list
=
[
7
,
10
,
15
]
def
__init__
(
self
,
particleContainer
,
desiredFraction
:
float
=
0.1
)
->
None
:
def
__init__
(
self
,
particleContainer
,
desiredFraction
:
float
=
0.1
)
->
None
:
super
(
Spiral
Selector
,
self
).
__init__
(
particleContainer
,
desiredFraction
)
super
(
Spiral
BoxSubsampling
,
self
).
__init__
(
particleContainer
,
desiredFraction
)
self
.
numBoxes
=
2
0
self
.
numBoxes
=
1
0
@
property
@
property
def
label
(
self
)
->
str
:
def
label
(
self
)
->
str
:
...
...
gui/measureModes.py
View file @
229dd4ac
from
PyQt5
import
QtCore
,
QtWidgets
from
PyQt5
import
QtCore
,
QtWidgets
from
gui.filterView
import
FilterView
,
MeasureBoxGraphItem
from
gui.filterView
import
FilterView
,
MeasureBoxGraphItem
from
geometricMethods
import
BoxSelectionSubsamplingMethod
,
CrossBoxS
elector
,
SpiralSelector
from
geometricMethods
import
BoxSelectionSubsamplingMethod
,
CrossBoxS
ubSampling
,
SpiralBoxSubsampling
class
MeasureMode
(
QtCore
.
QObject
):
class
MeasureMode
(
QtCore
.
QObject
):
...
@@ -26,7 +26,7 @@ class CrossBoxMode(MeasureMode):
...
@@ -26,7 +26,7 @@ class CrossBoxMode(MeasureMode):
def
__init__
(
self
,
*
args
):
def
__init__
(
self
,
*
args
):
super
(
CrossBoxMode
,
self
).
__init__
(
*
args
)
super
(
CrossBoxMode
,
self
).
__init__
(
*
args
)
self
.
uiControls
=
CrossBoxesControls
(
self
)
self
.
uiControls
=
CrossBoxesControls
(
self
)
self
.
boxGenerator
:
CrossBoxS
elector
=
CrossBoxS
elector
(
None
)
self
.
boxGenerator
:
CrossBoxS
ubSampling
=
CrossBoxS
ubSampling
(
None
)
self
.
update_measure_viewItems
()
self
.
update_measure_viewItems
()
def
update_measure_viewItems
(
self
)
->
None
:
def
update_measure_viewItems
(
self
)
->
None
:
...
@@ -91,7 +91,7 @@ class SpiralBoxMode(MeasureMode):
...
@@ -91,7 +91,7 @@ class SpiralBoxMode(MeasureMode):
def
__init__
(
self
,
*
args
):
def
__init__
(
self
,
*
args
):
super
(
SpiralBoxMode
,
self
).
__init__
(
*
args
)
super
(
SpiralBoxMode
,
self
).
__init__
(
*
args
)
self
.
uiControls
:
SpiralBoxControls
=
SpiralBoxControls
(
self
)
self
.
uiControls
:
SpiralBoxControls
=
SpiralBoxControls
(
self
)
self
.
boxGenerator
:
Spiral
Selector
=
SpiralSelector
(
None
)
self
.
boxGenerator
:
Spiral
BoxSubsampling
=
SpiralBoxSubsampling
(
None
)
self
.
update_measure_viewItems
()
self
.
update_measure_viewItems
()
def
update_measure_viewItems
(
self
)
->
None
:
def
update_measure_viewItems
(
self
)
->
None
:
...
...
helpers.py
View file @
229dd4ac
...
@@ -32,6 +32,24 @@ class ParticleBinSorter(object):
...
@@ -32,6 +32,24 @@ class ParticleBinSorter(object):
return
binIndex
return
binIndex
def
get_Anger_fraction
(
numParticles
,
sigma
=
1.65
,
mpFraction
=
0.01
,
errorMargin
=
0.1
):
"""
Returns the required fraction for reaching the defined errorMargin at a given number of Particles.
According to: Anger et al. "Raman microspectroscopy as a tool for microplastic particle analysis",
TrAC Trends in Analytical Chemistry, 2018, 214-226. (https://doi.org/10.1016/j.trac.2018.10.010)
:param numParticles:
:param sigma:
:param mpFraction:
:param errorMargin:
:return:
"""
N
=
numParticles
P
=
mpFraction
e
=
P
*
errorMargin
numParticlesMeasured
=
np
.
ceil
(
P
*
(
1
-
P
)
/
(
e
**
2
/
sigma
**
2
+
P
*
(
1
-
P
)
/
N
))
return
np
.
int
(
numParticlesMeasured
)
def
box_overlaps_contour
(
boxTopLeftXY
:
tuple
,
boxWidthHeight
:
tuple
,
contourData
:
np
.
ndarray
)
->
bool
:
def
box_overlaps_contour
(
boxTopLeftXY
:
tuple
,
boxWidthHeight
:
tuple
,
contourData
:
np
.
ndarray
)
->
bool
:
"""
"""
Calculates, if a contour is overlapping a box.
Calculates, if a contour is overlapping a box.
...
...
methods.py
View file @
229dd4ac
...
@@ -47,31 +47,6 @@ class RandomSampling(SubsamplingMethod):
...
@@ -47,31 +47,6 @@ class RandomSampling(SubsamplingMethod):
def
_get_number_of_random_particles
(
self
,
numTotalParticles
):
def
_get_number_of_random_particles
(
self
,
numTotalParticles
):
return
np
.
int
(
np
.
ceil
(
numTotalParticles
*
self
.
fraction
))
return
np
.
int
(
np
.
ceil
(
numTotalParticles
*
self
.
fraction
))
class
IvlevaSubsampling
(
SubsamplingMethod
):
def
__init__
(
self
,
particleContainer
,
sigma
=
1.65
,
mpFraction
=
0.01
,
errorMargin
=
0.1
):
super
(
IvlevaSubsampling
,
self
).
__init__
(
particleContainer
)
self
.
sigma
=
sigma
self
.
estimatedMPFraction
=
mpFraction
self
.
errorMargin
=
errorMargin
@
property
def
label
(
self
)
->
str
:
return
'Random fraction, Anger et al.'
def
apply_subsampling_method
(
self
)
->
list
:
N
=
self
.
particleContainer
.
getNumberOfParticles
()
numParticlesMeasured
=
self
.
_get_ivleva_fraction
(
N
)
subParticles
=
random
.
sample
(
self
.
particleContainer
.
particles
,
numParticlesMeasured
)
fractionMeasured
=
numParticlesMeasured
/
N
return
subParticles
def
_get_ivleva_fraction
(
self
,
N
):
P
=
self
.
estimatedMPFraction
e
=
P
*
self
.
errorMargin
numParticlesMeasured
=
np
.
ceil
(
P
*
(
1
-
P
)
/
(
e
**
2
/
self
.
sigma
**
2
+
P
*
(
1
-
P
)
/
N
))
return
np
.
int
(
numParticlesMeasured
)
class
SizeBinFractioning
(
SubsamplingMethod
):
class
SizeBinFractioning
(
SubsamplingMethod
):
...
...
subsampling.py
View file @
229dd4ac
...
@@ -6,10 +6,10 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
...
@@ -6,10 +6,10 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
from
gepard
import
dataset
from
gepard
import
dataset
import
gepardevaluation
import
gepardevaluation
from
methods
import
IvlevaSubsampling
,
RandomSampling
,
SizeBinFractioning
from
methods
import
RandomSampling
,
SizeBinFractioning
from
geometricMethods
import
BoxSelectionCreator
from
geometricMethods
import
BoxSelectionCreator
from
helpers
import
ParticleBinSorter
from
helpers
import
ParticleBinSorter
from
evaluation
import
ResultComparer
,
ResultObject
from
evaluation
import
TotalResults
"""
"""
...
@@ -25,70 +25,11 @@ workingFiles.append(r'C:\Users\xbrjos\Desktop\temp MP\190201_BSB_Stroomi_ds2_R1_
...
@@ -25,70 +25,11 @@ workingFiles.append(r'C:\Users\xbrjos\Desktop\temp MP\190201_BSB_Stroomi_ds2_R1_
# These do not work, due to no ramanscansortindex??
# These do not work, due to no ramanscansortindex??
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl'
# fname: str = r'C:\Users\xbrjos\Desktop\temp MP\KWS_CT_3_ds1_all_10_2\KWS_CT_3_ds1_all_10_2.pkl'
for
index
,
fname
in
enumerate
(
workingFiles
):
# dset = dataset.loadData(fname)
newObj
:
ResultObject
=
ResultObject
(
fname
)
# print('loaded dataset', fname)
# boxCreator = BoxSelectionCreator(dset)
# center, size = boxCreator.get_filterDimensions_from_dataset()
# print(center, size)
# print(dset.mapToPixel(center, force=True))
# pc = dset.particleContainer
# origParticles = pc.particles
# resultComparer = ResultComparer()
# numOrigMP = resultComparer._get_number_of_MP_particles(origParticles)
# print(f'orig particles: {len(origParticles)}, of which are mp: {numOrigMP}')
# # ivlevaSampling = IvlevaSubsampling(pc)
# # ivlevaFraction, ivlevaParticles = ivlevaSampling.apply_subsampling_method()
# t0 = time.time()
results
:
TotalResults
=
TotalResults
()
# fractions = np.arange(0.05, .55, 0.05)
# errors = []
# binErrors = []
# numIterations = 1000
# for fraction in fractions:
for
index
,
fname
in
enumerate
(
workingFiles
):
# print('random sampling, fraction:', fraction)
results
.
add_sample
(
fname
)
# # randomSampling = RandomSampling(pc, desiredFraction=fraction)
t0
=
time
.
time
()
# randomSampling = SizeBinFractioning(pc, fraction)
results
.
update_all
()
# iterErrors = []
print
(
'updating all took'
,
time
.
time
()
-
t0
,
'seconds'
)
# binIterErrors = []
# for _ in range(numIterations):
# randomFraction, randomParticles = randomSampling.apply_subsampling_method()
# iterErrors.append(resultComparer._get_mp_count_error(origParticles, randomParticles, randomFraction))
# bins, errorsPerBin = resultComparer._get_mp_count_error_per_bin(origParticles, randomParticles, randomFraction)
# binIterErrors.append(errorsPerBin)
# errors.append(round(np.mean(iterErrors)*100)) #from fraction to %
# fractionBinErrors = []
# for binIndex in range(len(bins)+1):
# binError = round(np.mean([binIterErrors[i][binIndex] for i in range(numIterations)]) * 100)
# fractionBinErrors.append(binError)
# binErrors.append(fractionBinErrors)
# print('random sampling took', np.round(time.time()-t0, 2), 'seonds')
# binLowerLimits = bins.copy()
# binLowerLimits.insert(0, 0)
# plt.subplot(121)
# plt.plot(fractions, errors)
# # plt.title(f'Random Sampling, averaged from {numIterations} trials, orig particle count: {len(origParticles)}')
# plt.xlabel('Fraction measured')
# plt.ylabel('Average error in MP particle count (%)')
# plt.subplot(122)
# for fracMeas, curBinErrors in zip(fractions, binErrors):
# plt.plot(binLowerLimits, curBinErrors, label=np.round(fracMeas, 1))
# # plt.title('Error in MP count (%) per size bin')
# plt.xlabel('particle size')
# plt.ylabel('Average error in MP particle count (%)')
# plt.legend()
# plt.show()
# # sizeBinSampling = SizeBinFractioning(pc)
# # sizeBinParticles = sizeBinSampling.apply_subsampling_method()
\ No newline at end of file
tests/test_evaluation.py
View file @
229dd4ac
...
@@ -13,52 +13,172 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
...
@@ -13,52 +13,172 @@ sys.path.append("C://Users//xbrjos//Desktop//Python")
import
gepard
import
gepard
from
gepard.analysis.particleAndMeasurement
import
Particle
,
Measurement
from
gepard.analysis.particleAndMeasurement
import
Particle
,
Measurement
from
evaluation
import
ResultComparer
from
evaluation
import
TotalResults
,
SampleResult
,
SubsamplingResult
import
methods
as
meth
import
geometricMethods
as
gmeth
class
TestResultComparer
(
unittest
.
TestCase
):
class
TestTotalResults
(
unittest
.
TestCase
):
def
setUp
(
self
)
->
None
:
self
.
totalResults
=
TotalResults
()
def
test_add_sample
(
self
):
self
.
totalResults
.
add_sample
(
'fakePath/fakeFolder/fakeFile.pkl'
)
self
.
assertEqual
(
len
(
self
.
totalResults
.
sampleResults
),
1
)
self
.
totalResults
.
add_sample
(
'fakePath/fakeFolder/fakeFile.pkl'
)
# the same file should not be added again
self
.
assertEqual
(
len
(
self
.
totalResults
.
sampleResults
),
1
)
self
.
totalResults
.
add_sample
(
'fakePath/fakeFolder/fakeFile2.pkl'
)
# another should be added, though
self
.
assertEqual
(
len
(
self
.
totalResults
.
sampleResults
),
2
)
self
.
totalResults
.
add_sample
(
'fakePath/fakeFolder/fakeFile2.txt'
)
# invalid extention, not added...
self
.
assertEqual
(
len
(
self
.
totalResults
.
sampleResults
),
2
)
def
test_get_methods_for_fraction
(
self
):
def
containsMethod
(
listOfMethods
:
list
,
template
:
meth
.
SubsamplingMethod
)
->
bool
:
contains
:
bool
=
False
for
method
in
listOfMethods
:
if
type
(
method
)
==
type
(
template
)
and
method
.
fraction
==
template
.
fraction
:
contains
=
True
break
return
contains
dset
:
gepard
.
dataset
.
DataSet
=
gepard
.
dataset
.
DataSet
(
'fakepath'
)
imgdim
=
10
dset
.
imagescanMode
=
'df'
dset
.
imagedim_df
=
[
imgdim
,
imgdim
]
dset
.
pixelscale_df
=
1.0
minX
,
maxX
,
minY
,
maxY
=
0
,
1000
,
0
,
1000
dset
.
maxdim
=
minX
+
imgdim
/
2
,
maxY
-
imgdim
/
2
,
maxX
-
imgdim
/
2
,
minY
+
imgdim
/
2
desiredFraction
=
0.1
methods
=
self
.
totalResults
.
_get_methods_for_fraction
(
dset
,
desiredFraction
)
possibleRandomMethods
=
2
possibleCrossBoxMethods
=
2