Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
S
Subsampling
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Package Registry
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Josef Brandt
Subsampling
Commits
ee19bc2c
Commit
ee19bc2c
authored
Mar 20, 2020
by
Josef Brandt
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Kennard Stone Optimization, but now bug??
parent
5d22e3c7
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
58 additions
and
25 deletions
+58
-25
chemometricMethods.py
chemometricMethods.py
+15
-6
cythonModules/kennardStone.pyx
cythonModules/kennardStone.pyx
+11
-5
evaluation.py
evaluation.py
+4
-1
tests/test_chemometricMethods.py
tests/test_chemometricMethods.py
+19
-10
tests/test_evaluation.py
tests/test_evaluation.py
+9
-3
No files found.
chemometricMethods.py
View file @
ee19bc2c
...
...
@@ -24,16 +24,18 @@ class ChemometricSubsampling(SubsamplingMethod):
def
__init__
(
self
,
particleContainer
:
ParticleContainer
,
desiredFraction
:
float
):
super
(
ChemometricSubsampling
,
self
).
__init__
(
particleContainer
,
desiredFraction
)
@
property
def
label
(
self
)
->
str
:
return
'Chemometric Selection'
def
apply_subsampling_method
(
self
)
->
list
:
vectors
:
np
.
ndarray
=
self
.
_get_particle_featurematrix
()
kennardStone
:
KennardStone
=
KennardStone
(
vectors
,
self
.
fraction
)
princComps
:
np
.
ndarray
=
get_pca
(
vectors
)
kennardStone
:
KennardStone
=
KennardStone
(
princComps
,
self
.
fraction
)
indices
:
list
=
kennardStone
.
get_sampled_indices
()
selectedParticles
:
list
=
[]
for
particle
in
self
.
particleContainer
.
particles
:
if
particle
.
index
in
selectedParticl
es
:
if
particle
.
index
in
indic
es
:
selectedParticles
.
append
(
particle
)
return
selectedParticles
...
...
@@ -46,6 +48,12 @@ class ChemometricSubsampling(SubsamplingMethod):
vectors
:
np
.
ndarray
=
np
.
transpose
(
np
.
array
(
vectors
))
return
vectors
def
equals
(
self
,
otherMethod
)
->
bool
:
equals
:
bool
=
False
if
type
(
otherMethod
)
==
ChemometricSubsampling
and
otherMethod
.
fraction
==
self
.
fraction
:
equals
=
True
return
equals
class
FeatureExtractor
(
object
):
def
__init__
(
self
,
particle
:
pm
.
Particle
):
...
...
@@ -78,9 +86,10 @@ class KennardStone(object):
Adapted from https://github.com/karoka/Kennard-Stone-Algorithm/blob/master/kenStone.py
:return"""
t0
=
time
.
time
()
numIndices
:
int
=
round
(
self
.
data
.
shape
[
0
]
*
self
.
fraction
)
if
numIndices
<
2
:
raise
ValueError
(
f
'Not enough indices to generate (min = 2), requested:
{
numIndices
}
'
)
assert
self
.
data
.
shape
[
1
]
==
2
numIndices
:
int
=
int
(
np
.
ceil
(
self
.
data
.
shape
[
0
]
*
self
.
fraction
))
if
numIndices
==
1
:
selectedIndices
=
[
int
(
round
(
self
.
data
.
shape
[
0
]
/
2
))]
else
:
distMat
=
spatial
.
distance_matrix
(
self
.
data
,
self
.
data
)
...
...
@@ -88,7 +97,7 @@ class KennardStone(object):
selectedIndices
=
find_furthest_indices
(
distMat
,
int
(
numIndices
),
i
,
j
)
assert
len
(
np
.
unique
(
list
(
selectedIndices
)))
==
len
(
selectedIndices
)
print
(
'selecting indices time:'
,
np
.
round
(
time
.
time
()
-
t0
,
2
),
'seconds
'
)
print
(
f
'selecting indices time:
{
np
.
round
(
time
.
time
()
-
t0
,
2
)
}
seconds (
{
numIndices
}
out of
{
self
.
data
.
shape
[
0
]
}
)
'
)
return
list
(
selectedIndices
)
def
_get_start_indices
(
self
)
->
list
:
...
...
cythonModules/kennardStone.pyx
View file @
ee19bc2c
...
...
@@ -9,20 +9,25 @@ ctypedef np.int64_t INT64_t
def
find_furthest_indices
(
np
.
ndarray
[
DTYPE_t
,
ndim
=
2
]
distMat
,
int
numIndices
,
int
index0
,
int
index1
):
cdef
int
i
,
j
cdef
int
i
,
j
,
jidx
,
kidx
cdef
double
dist
,
minDist
,
curDist
cdef
np
.
ndarray
[
INT32_t
,
ndim
=
1
]
selectedIndices
=
np
.
empty
(
numIndices
,
dtype
=
np
.
int32
)
cdef
np
.
ndarray
[
INT32_t
,
ndim
=
1
]
remainingIndices
=
np
.
arange
(
numIndices
,
dtype
=
np
.
int32
)
cdef
int
numSelectedIndices
=
2
cdef
int
numRemainingIndices
=
distMat
.
shape
[
0
]
-
2
selectedIndices
[
0
]
=
index0
selectedIndices
[
1
]
=
index1
for
i
in
range
(
numIndices
-
2
):
minDist
=
0.0
for
j
in
remainingIndices
:
dist
=
np
.
inf
for
jidx
in
range
(
remainingIndices
.
shape
[
0
]):
j
=
remainingIndices
[
jidx
]
dist
=
1E6
for
k
in
selectedIndices
[:
i
+
1
]:
curDist
=
distMat
[
j
][
k
]
for
kidx
in
range
(
numSelectedIndices
):
k
=
selectedIndices
[
kidx
]
curDist
=
distMat
[
j
,
k
]
if
curDist
<
dist
:
dist
=
curDist
...
...
@@ -31,6 +36,7 @@ def find_furthest_indices(np.ndarray[DTYPE_t, ndim=2] distMat, int numIndices, i
minDist
=
dist
selectedIndices
[
i
+
2
]
=
minj
numSelectedIndices
+=
1
remainingIndices
=
remainingIndices
[
remainingIndices
!=
minj
]
return
selectedIndices
\ No newline at end of file
evaluation.py
View file @
ee19bc2c
...
...
@@ -14,6 +14,7 @@ import matplotlib.pyplot as plt
from
helpers
import
ParticleBinSorter
import
methods
as
meth
import
geometricMethods
as
gmeth
import
chemometricMethods
as
cmeth
sys
.
path
.
append
(
"C://Users//xbrjos//Desktop//Python"
)
from
gepard
import
dataset
...
...
@@ -25,7 +26,7 @@ def get_name_from_directory(dirPath: str) -> str:
class
TotalResults
(
object
):
methods
:
list
=
[
meth
.
RandomSampling
,
meth
.
SizeBinFractioning
,
gmeth
.
CrossBoxSubSampling
,
gmeth
.
SpiralBoxSubsampling
]
measuredFreactions
:
list
=
[
0.05
,
0.1
,
0.15
,
0.2
,
0.5
,
0.75
,
0.9
]
measuredFreactions
:
list
=
[
0.0
1
,
0.0
5
,
0.1
,
0.15
,
0.2
,
0.5
,
0.75
,
0.9
]
def
__init__
(
self
):
super
(
TotalResults
,
self
).
__init__
()
...
...
@@ -107,6 +108,8 @@ class TotalResults(object):
boxCreator
:
gmeth
.
BoxSelectionCreator
=
gmeth
.
BoxSelectionCreator
(
dataset
)
methods
+=
boxCreator
.
get_crossBoxSubsamplers_for_fraction
(
fraction
)
methods
+=
boxCreator
.
get_spiralBoxSubsamplers_for_fraction
(
fraction
)
if
fraction
<=
0.1
:
methods
.
append
(
cmeth
.
ChemometricSubsampling
(
particleContainer
,
fraction
))
return
methods
...
...
tests/test_chemometricMethods.py
View file @
ee19bc2c
...
...
@@ -53,7 +53,7 @@ class TestKennardStone(unittest.TestCase):
self
.
kennardStone
:
cmeth
.
KennardStone
=
cmeth
.
KennardStone
(
np
.
array
([]),
0.1
)
def
test_get_sampled_indices
(
self
):
numDataSets
:
int
=
1
000
numDataSets
:
int
=
4
000
self
.
kennardStone
.
data
=
np
.
random
.
rand
(
numDataSets
,
2
)
self
.
kennardStone
.
fraction
=
0.1
selectedIndices
=
self
.
kennardStone
.
get_sampled_indices
()
...
...
@@ -63,10 +63,19 @@ class TestKennardStone(unittest.TestCase):
self
.
kennardStone
.
fraction
=
0.1
numDataSets
=
2
self
.
kennardStone
.
data
=
np
.
random
.
rand
(
numDataSets
,
2
)
self
.
assertRaises
(
ValueError
,
self
.
kennardStone
.
get_sampled_indices
)
selectedIndices
=
self
.
kennardStone
.
get_sampled_indices
()
self
.
assertEqual
(
len
(
selectedIndices
),
1
)
self
.
kennardStone
.
fraction
=
0.5
numDataSets
=
2
self
.
kennardStone
.
data
=
np
.
random
.
rand
(
numDataSets
,
2
)
selectedIndices
=
self
.
kennardStone
.
get_sampled_indices
()
self
.
assertEqual
(
len
(
selectedIndices
),
numDataSets
*
self
.
kennardStone
.
fraction
)
self
.
assertEqual
(
len
(
np
.
unique
(
selectedIndices
)),
numDataSets
*
self
.
kennardStone
.
fraction
)
numDataSets
=
20
self
.
kennardStone
.
data
=
np
.
random
.
rand
(
numDataSets
,
2
)
self
.
kennardStone
.
fraction
=
0.1
selectedIndices
=
self
.
kennardStone
.
get_sampled_indices
()
self
.
assertEqual
(
len
(
selectedIndices
),
2
)
self
.
assertEqual
(
len
(
np
.
unique
(
selectedIndices
)),
2
)
...
...
@@ -120,13 +129,13 @@ class TestChemometricSubsampling(unittest.TestCase):
particleContainer
.
setParticleContours
([
contours
[
0
]
for
i
in
range
(
numParticles
)])
self
.
chemSubs
:
cmeth
.
ChemometricSubsampling
=
cmeth
.
ChemometricSubsampling
(
particleContainer
,
desiredFraction
=
0.1
)
def
test_get_particle_featurematrix
(
self
):
features
:
np
.
ndarray
=
self
.
chemSubs
.
_get_particle_featurematrix
()
self
.
assertEqual
(
features
.
shape
,
(
7
,
5
))
for
i
in
range
(
6
):
diff
:
np
.
ndarray
=
features
[
i
,
:]
-
np
.
mean
(
features
[
i
,
:])
self
.
assertFalse
(
np
.
any
(
diff
>
0.1
))
#
def test_get_particle_featurematrix(self):
#
features: np.ndarray = self.chemSubs._get_particle_featurematrix()
#
self.assertEqual(features.shape, (7, 5))
#
for i in range(6):
#
diff: np.ndarray = features[i, :] - np.mean(features[i, :])
#
self.assertFalse(np.any(diff > 0.1))
# def test_pca(self):
# fname = r'C:\Users\xbrjos\Desktop\temp MP\190326_MCII_WWTP_SB_50_1\190326_MCII_WWTP_SB_50_1.pkl'
# fname = r'C:\Users\xbrjos\Desktop\temp MP\190313_Soil_5_A_50_5_1_50_1\190313_Soil_5_A_50_5_1_50_1.pkl'
...
...
@@ -137,7 +146,7 @@ class TestChemometricSubsampling(unittest.TestCase):
# plt.scatter(princComp[:, 0], princComp[:, 1])
# plt.title(dset.name)
# plt.show()
#
# class TestBAH(unittest.TestCase):
# # def setUp(self) -> None:
...
...
tests/test_evaluation.py
View file @
ee19bc2c
...
...
@@ -62,7 +62,9 @@ class TestTotalResults(unittest.TestCase):
possibleRandomMethods
=
2
possibleCrossBoxMethods
=
2
possibleSpiralBoxMethods
=
3
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
possibleSpiralBoxMethods
possibleChemometricMethods
=
1
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
\
possibleSpiralBoxMethods
+
possibleChemometricMethods
self
.
assertEqual
(
len
(
methods
),
totalPossible
)
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
RandomSampling
(
dset
,
desiredFraction
)))
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
SizeBinFractioning
(
dset
,
desiredFraction
)))
...
...
@@ -74,7 +76,9 @@ class TestTotalResults(unittest.TestCase):
possibleRandomMethods
=
2
possibleCrossBoxMethods
=
1
possibleSpiralBoxMethods
=
0
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
possibleSpiralBoxMethods
possibleChemometricMethods
=
0
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
\
possibleSpiralBoxMethods
+
possibleChemometricMethods
self
.
assertEqual
(
len
(
methods
),
totalPossible
)
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
RandomSampling
(
dset
,
desiredFraction
)))
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
SizeBinFractioning
(
dset
,
desiredFraction
)))
...
...
@@ -86,7 +90,9 @@ class TestTotalResults(unittest.TestCase):
possibleRandomMethods
=
2
possibleCrossBoxMethods
=
0
possibleSpiralBoxMethods
=
0
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
possibleSpiralBoxMethods
possibleChemometricMethods
=
0
totalPossible
=
possibleCrossBoxMethods
+
possibleRandomMethods
+
\
possibleSpiralBoxMethods
+
possibleChemometricMethods
self
.
assertEqual
(
len
(
methods
),
totalPossible
)
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
RandomSampling
(
dset
,
desiredFraction
)))
self
.
assertTrue
(
containsMethod
(
methods
,
meth
.
SizeBinFractioning
(
dset
,
desiredFraction
)))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment