Commit 900f1adb authored by Robert's avatar Robert Committed by Robert Ohmacht

-log files will now be rotated after reaching 5 MB, up to 10 times

-some additional logging
-fixed some index confusions (cell/pixel indexing in cv2 vs numpy) in ScenePyramid
parent 1231ca15
......@@ -19,6 +19,7 @@ along with this program, see COPYING.
If not, see <https://www.gnu.org/licenses/>.
"""
import logging
import logging.handlers
import traceback
import os
from io import StringIO
......@@ -358,10 +359,8 @@ if __name__ == '__main__':
sys.excepthook = excepthook
logger = logging.getLogger(__name__)
setDefaultLoggingConfig(logger)
app = QtWidgets.QApplication(sys.argv)
app.setApplicationName("GEPARD") # appname needed for logpath
app.setApplicationName("GEPARD") # appname needed for logpath
logpath = QtCore.QStandardPaths.writableLocation(
QtCore.QStandardPaths.AppLocalDataLocation)
......@@ -370,8 +369,12 @@ if __name__ == '__main__':
if not os.path.exists(logpath):
os.mkdir(logpath)
logname = os.path.join(logpath, 'logfile.txt')
logger.addHandler(logging.FileHandler(logname))
logger.addHandler(
logging.handlers.RotatingFileHandler(
logname, maxBytes=5*(1 << 20), backupCount=10)
)
setDefaultLoggingConfig(logger)
logger.info("starting GEPARD at: " + strftime("%d %b %Y %H:%M:%S", localtime()))
gepard = GEPARDMainWindow(logger)
......
......@@ -21,6 +21,7 @@ If not, see <https://www.gnu.org/licenses/>.
import numpy as np
import os
import logging
import logging.handlers
from PyQt5 import QtCore, QtWidgets, QtGui
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
import matplotlib.pyplot as plt
......@@ -288,7 +289,10 @@ class ParticleDetectionView(QtWidgets.QWidget):
logPath = os.path.join(self.dataset.path, 'detectionlog.txt')
self.logger = logging.getLogger('detection')
self.logger.addHandler(logging.FileHandler(logPath))
self.logger.addHandler(
logging.handlers.RotatingFileHandler(
logPath, maxBytes=5*(1 << 20), backupCount=10)
)
setDefaultLoggingConfig(self.logger)
vbox = QtWidgets.QVBoxLayout()
......@@ -779,7 +783,10 @@ class ParticleDetectionView(QtWidgets.QWidget):
def _worker(self):
logPath = os.path.join(self.dataset.path, 'detectionlog.txt')
detectLogger = logging.getLogger('detection_worker')
detectLogger.addHandler(logging.FileHandler(logPath))
detectLogger.addHandler(
logging.handlers.RotatingFileHandler(
logPath, maxBytes=5 * (1 << 20), backupCount=10)
)
setDefaultLoggingConfig(detectLogger)
kwargs = {}
......
......@@ -21,6 +21,7 @@ If not, see <https://www.gnu.org/licenses/>.
import logging
def setDefaultLoggingConfig(logger: logging.Logger):
"""
Applies a default configuration to the specified logger
......@@ -41,4 +42,4 @@ def setDefaultLoggingConfig(logger: logging.Logger):
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
for handler in logger.handlers:
handler.setFormatter(formatter)
\ No newline at end of file
handler.setFormatter(formatter)
......@@ -26,6 +26,7 @@ import queue
from .imagestitch import imageStacking
import os, sys
import logging
import logging.handlers
import cv2
from .helperfunctions import cv2imread_fix, cv2imwrite_fix
from time import time, localtime, strftime
......@@ -35,13 +36,17 @@ from .zlevelsetter import ZLevelSetter
from .scenePyramid import ScenePyramid
from .gepardlogging import setDefaultLoggingConfig
def scan(path, sol, zpositions, grid, controlclass, dataqueue,
stopevent, logpath, ishdr=False):
if ishdr:
merge_mertens = cv2.createMergeMertens()
logger = logging.getLogger()
logger.addHandler(logging.FileHandler(logpath))
logger.addHandler(
logging.handlers.RotatingFileHandler(
logpath, maxBytes=5 * (1 << 20), backupCount=10)
)
setDefaultLoggingConfig(logger)
logger.info('starting new optical scan')
......@@ -65,13 +70,16 @@ def scan(path, sol, zpositions, grid, controlclass, dataqueue,
values = [5.,25.,100.]
for j, val in enumerate(values if (i%2+k%2)%2==0 else reversed(values)):
ramanctrl.setBrightness(val)
logger.info(f'writing hdr image to {fname}')
ramanctrl.saveImage(fname)
img_list.append(cv2imread_fix(fname))
res_mertens = merge_mertens.process(img_list)
res_mertens_8bit = np.clip(res_mertens*255, 0, 255).astype('uint8')
cv2imwrite_fix(os.path.join(path,name), res_mertens_8bit)
cv2imwrite_fix(os.path.join(path, name), res_mertens_8bit)
else:
ramanctrl.saveImage(os.path.join(path,name))
img_path = os.path.join(path, name)
logger.info(f'writing non hdr image to {img_path}')
ramanctrl.saveImage(img_path)
if stopevent.is_set():
ramanctrl.disconnect()
return
......@@ -88,14 +96,17 @@ def subtractBackground(image, background):
subtracted = np.clip(np.array(image - background + avg, dtype=np.uint8), 0, 255)
return subtracted
def legacyLoadAndPasteImage(srcnames, fullimage, fullzval, width, height,
rotationvalue, p0, p1, p, background=None):
rotationvalue, p0, p1, p, background=None):
colimgs = []
for name in srcnames:
curImg = cv2imread_fix(name)
if background is not None:
curImg = subtractBackground(curImg, background)
colimgs.append(curImg)
if os.path.isfile(name):
curImg = cv2imread_fix(name)
if background is not None:
curImg = subtractBackground(curImg, background)
colimgs.append(curImg)
img, zval = imageStacking(colimgs)
x, y = p
......@@ -115,6 +126,7 @@ def legacyLoadAndPasteImage(srcnames, fullimage, fullzval, width, height,
zval = cv2.warpAffine(zval, M, (Nx, Ny))
return dst, zval
def removeSrcTiles(names, path):
files = os.listdir(path)
for fpath in names:
......@@ -124,7 +136,7 @@ def removeSrcTiles(names, path):
def loadAndPasteImage(srcnames, pyramid, fullzval, width, height,
rotationvalue, p0, p1, p, background=None):
rotationvalue, p0, p1, p, logger, background=None):
"""
:param list of str srcnames: list of stacked scan files to merge
:param ScenePyramid pyramid: the scene pyramid
......@@ -140,10 +152,15 @@ def loadAndPasteImage(srcnames, pyramid, fullzval, width, height,
"""
colimgs = []
for name in srcnames:
curImg = cv2imread_fix(name)
if background is not None:
curImg = subtractBackground(curImg, background)
colimgs.append(curImg)
if os.path.isfile(name):
curImg = cv2imread_fix(name)
if background is not None:
curImg = subtractBackground(curImg, background)
colimgs.append(curImg)
if 0 == len(colimgs):
logger.warning(f'no src image(s) found to paste in loadAndPasteImage()')
img, zval = imageStacking(colimgs)
x, y = p
......@@ -723,7 +740,7 @@ class OpticalScan(QtWidgets.QWidget):
self.view.imgdata, self.dataset.zvalimg = loadAndPasteImage(
names, self.pyramid, self.dataset.zvalimg, width, height,
rotationvalue, p0, p1, p, background=background_img
rotationvalue, p0, p1, p, self.logger, background=background_img
)
removeSrcTiles(names, self.dataset.getScanPath())
......
......@@ -26,6 +26,7 @@ import queue
from time import time, localtime, strftime
import os
import logging
import logging.handlers
from .external import tsp
from .uielements import TimeEstimateProgressbar
from .gepardlogging import setDefaultLoggingConfig
......@@ -70,7 +71,10 @@ def scan(ramanSettings, positions, controlclass, dataqueue, stopevent,
logpath=''):
if logpath != '':
logger = logging.getLogger('RamanScanLogger')
logger.addHandler(logging.FileHandler(logpath))
logger.addHandler(
logging.handlers.RotatingFileHandler(
logpath, maxBytes=5 * (1 << 20), backupCount=10)
)
setDefaultLoggingConfig(logger)
try:
......
......@@ -23,6 +23,7 @@ import numpy as np
import os
import time
import logging
import logging.handlers
from .dataset import DataSet, loadData
from .ramancom.ramancontrol import RamanControl, simulatedRaman
from .opticalscan import OpticalScan
......@@ -35,6 +36,7 @@ from .analysis.colorlegend import getColorFromNameWithSeed
from .analysis.particleEditor import ParticleEditor
from .ramancom.configRaman import RamanConfigWin
from .scenePyramid import ScenePyramid
from .gepardlogging import setDefaultLoggingConfig
class SampleView(QtWidgets.QGraphicsView):
......@@ -282,8 +284,12 @@ class SampleView(QtWidgets.QGraphicsView):
:return:
"""
logPath = os.path.join(self.dataset.path, 'gepardLog.txt')
self.logger.info(f'creating new log hander to path: {logPath}')
self.logger.addHandler(logging.FileHandler(logPath))
self.logger.info(f'creating new log handler to path: {logPath}')
self.logger.addHandler(
logging.handlers.RotatingFileHandler(
logPath, maxBytes=5 * (1 << 20), backupCount=10)
)
setDefaultLoggingConfig(self.logger)
def setupParticleEditor(self):
"""
......
......@@ -45,7 +45,7 @@ class ScenePyramid:
return {
'preScanModeComplete': False,
'maxSliceNumber': 0,
'tileDim': (1000, 1000),
'tileDim': (1000, 1000), # as np.ndarray.shape (i.e. (height, width))
'scalingFactor': .5,
'tileSets': {},
'fullImageWidth': 0,
......@@ -60,8 +60,10 @@ class ScenePyramid:
#
self.opacity = 1
self.xIdx = 0
self.yIdx = 1
self.npHeightIdx = 0 # np.ndarray.shape height idx
self.npWidthIdx = 1 # np.ndarray.shape width idx
self.cvHeightIdx = 1 # cv2 height idx
self.cvWidthIdx = 0 # cv2 width idx
self.currentSlice = None
self.microscopeMode = None
self.currentTiles = None
......@@ -69,7 +71,7 @@ class ScenePyramid:
self.preScanModeComplete = None
self.maxSliceNumber = None
self.tileDim = None
self.tileDim = None # as np.ndarray.shape (i.e. (height, width))
self.scalingFactor = None
self.tileSets = None
self.fullImageWidth = None
......@@ -145,7 +147,7 @@ class ScenePyramid:
scene_origin_x, scene_origin_y = (0, 0) # self.dataset.mapToPixel(p, self.microscopeMode, force=True)
# default width, height of view tiles
view_tile_width, view_tile_height = self.tileDim
view_tile_height, view_tile_width = self.tileDim
inv_scaling = (1 / self.scalingFactor) ** self.currentSlice
......@@ -254,8 +256,8 @@ class ScenePyramid:
scene_origin_x, scene_origin_y = (0, 0) # self.dataset.mapToPixel(p, self.microscopeMode, force=True)
# scaled position of currently added scan tile
pos_x = cur_scaling * scan_tile_pos[self.xIdx]
pos_y = cur_scaling * scan_tile_pos[self.yIdx]
pos_x = cur_scaling * scan_tile_pos[self.npHeightIdx]
pos_y = cur_scaling * scan_tile_pos[self.npWidthIdx]
# default width, height of view tiles
view_tile_width, view_tile_height = self.tileDim
......@@ -386,8 +388,8 @@ class ScenePyramid:
item.setScale(inv_scale_factor)
# and adjust position of tile accordingly
x += i * (inv_scale_factor - 1) * int(self.tileDim[self.xIdx])
y += j * (inv_scale_factor - 1) * int(self.tileDim[self.yIdx])
x += i * (inv_scale_factor - 1) * int(self.tileDim[self.npWidthIdx])
y += j * (inv_scale_factor - 1) * int(self.tileDim[self.npHeightIdx])
item.setPos(x, y)
......@@ -483,7 +485,7 @@ class ScenePyramid:
adds a src tile by pos and dimension
"""
item = QtWidgets.QGraphicsPixmapItem()
item.setPos(pos[self.xIdx], pos[self.yIdx])
item.setPos(pos[self.cvWidthIdx], pos[self.cvHeightIdx])
item.setAcceptedMouseButtons(QtCore.Qt.NoButton)
height, width, channel = img.shape
pix = QtGui.QPixmap()
......@@ -516,11 +518,11 @@ class ScenePyramid:
"""
self.preScanModeComplete = True
self.fullImageWidth = current_full_image_width = fullimgsize[self.xIdx]
self.fullImageHeight = current_full_image_height = fullimgsize[self.yIdx]
self.fullImageWidth = current_full_image_width = fullimgsize[self.cvWidthIdx]
self.fullImageHeight = current_full_image_height = fullimgsize[self.cvHeightIdx]
# width, height of view tiles
view_tile_width, view_tile_height = self.tileDim
view_tile_height, view_tile_width = self.tileDim
# dst pos of src tile
pos_x, pos_y = v_trans
......@@ -567,7 +569,7 @@ class ScenePyramid:
for j in range(jo, je + 1):
# add to v(i, j)
tile = self.readViewTile(slice_nr, i, j)
size = (tile.shape[1], tile.shape[0])
size = (tile.shape[self.npWidthIdx], tile.shape[self.npHeightIdx]) # (w, h)
# translation matrix
m = np.float32([
[1, 0, src_tile_pos_x],
......@@ -589,7 +591,7 @@ class ScenePyramid:
def saveViewTile(self, img, slice_nr, i, j):
"""
saves tile to file system
saves tile to file system and remembers the dimension as (width, height)
:param img:
:param slice_nr:
:param i:
......@@ -606,8 +608,14 @@ class ScenePyramid:
self.tileWorkingSets[slice_nr][i] = {}
self.tileSets[slice_nr][i] = {}
if j not in self.tileWorkingSets[slice_nr][i]:
self.tileWorkingSets[slice_nr][i][j] = {"dimensions": (img.shape[1], img.shape[0]), "rendered": False}
self.tileSets[slice_nr][i][j] = {"dimensions": (img.shape[1], img.shape[0]), "rendered": False}
self.tileWorkingSets[slice_nr][i][j] = {
"dimensions": (img.shape[self.npWidthIdx], img.shape[self.npHeightIdx]),
"rendered": False
}
self.tileSets[slice_nr][i][j] = {
"dimensions": (img.shape[self.npWidthIdx], img.shape[self.npHeightIdx]),
"rendered": False
}
def readViewTile(self, slice_nr, i, j):
"""
......@@ -620,7 +628,8 @@ class ScenePyramid:
"""
tile_path = os.path.join(self.dataset.getTilePath(), f"tile_{slice_nr}_{i}_{j}.tif")
if not os.path.exists(tile_path): #
if not os.path.exists(tile_path):
# fallback for datasets still using *.bmp
tile_path = os.path.join(self.dataset.getTilePath(), f"tile_{slice_nr}_{i}_{j}.bmp")
try:
......@@ -631,7 +640,7 @@ class ScenePyramid:
# as full image size might not be divisible by tile dimensions, we have to determine correct tile size
# tile numbering is 0 based
scaling = self.scalingFactor ** slice_nr
max_x, max_y = self.tileDim
max_y, max_x = self.tileDim
fiw = scaling * self.fullImageWidth
fih = scaling * self.fullImageHeight
......@@ -639,11 +648,13 @@ class ScenePyramid:
tile_h = int(math.ceil(max(min(max_y, fih - j * max_y), 0)))
# tile does exists but its shape is not correct
'''
if tile is not None and (tile_h, tile_w) != tile.shape[:2]:
# this should not happen
raise TileSizeError(
f"tile should have dim ({tile_w}; {tile_h}), but has dim ({tile.shape[1]}; {tile.shape[0]})"
)
'''
# tile does not even exist, create
if tile is None:
......@@ -662,7 +673,7 @@ class ScenePyramid:
for i in self.tileWorkingSets[0]:
for j in self.tileWorkingSets[0][i]:
timg = Image.fromarray(self.readViewTile(0, i, j))
pos = (i * self.tileDim[self.xIdx], j * self.tileDim[self.yIdx])
pos = (i * self.tileDim[self.npWidthIdx], j * self.tileDim[self.npHeightIdx]) # as (w, h)
fullsize_img.paste(timg, pos)
fullsize_img.save(self.dataset.getImageName(), 'tiff', save_all=True, compression='tiff_deflate')
......@@ -679,8 +690,8 @@ class ScenePyramid:
tile_base_path = dset.getTilePath()
p = ScenePyramid.getDefaults()
tile_x = p['tileDim'][0]
tile_y = p['tileDim'][1]
tile_x = p['tileDim'][1] # self.npWidthIdx
tile_y = p['tileDim'][0] # self.npHeightIdx
scaling = p['scalingFactor']
warnings.simplefilter('ignore', Image.DecompressionBombError)
......@@ -780,8 +791,8 @@ class ScenePyramid:
)
extract pixels
'''
w = self.tileDim[self.xIdx]
h = self.tileDim[self.yIdx]
w = self.tileDim[self.npWidthIdx]
h = self.tileDim[self.npHeightIdx]
imin = math.floor(p0x / w)
imax = math.floor(p1x / w)
jmin = math.floor(p0y / h)
......@@ -837,8 +848,8 @@ class ScenePyramid:
for y in range(len(self.tileWorkingSets[0][x])):
tile = self.readViewTile(0, x, y)
if 1. != scale:
w = math.floor(scale * tile.shape[1])
h = math.floor(scale * tile.shape[0])
w = math.floor(scale * tile.shape[self.npWidthIdx])
h = math.floor(scale * tile.shape[self.npHeightIdx])
tile = np.array(Image.fromarray(tile).resize((w, h), resample=Image.BICUBIC))
if not first_tile:
......@@ -877,8 +888,9 @@ class ScenePyramid:
:return:
"""
scaling = 1
tile_width, tile_height = self.tileDim
for slice in range(0, self.maxSliceNumber + 1):
tile_height, tile_width = self.tileDim
# for every slice
for pslice in range(0, self.maxSliceNumber + 1):
img_width = math.ceil(scaling * self.fullImageWidth)
img_height = math.ceil(scaling * self.fullImageHeight)
# tiles in x direction
......@@ -886,17 +898,19 @@ class ScenePyramid:
# tiles in y direction
row_tile_count = math.ceil(img_height / tile_height)
# col in x direction
for i in range(0, col_tile_count):
w_tile = (tile_width if i + 1 < col_tile_count else img_width % tile_width)
if i not in self.tileSets[slice]:
self.tileSets[slice][i] = {}
self.tileWorkingSets[slice][i] = {}
col = self.tileSets[slice][i]
if i not in self.tileSets[pslice]:
self.tileSets[pslice][i] = {}
self.tileWorkingSets[pslice][i] = {}
col = self.tileSets[pslice][i]
# row in y direction
for j in range(0, row_tile_count):
h_tile = (tile_height if j + 1 < row_tile_count else img_height % tile_height)
if j not in col or col[j]["dimensions"] != (w_tile, h_tile):
tile = self.readViewTile(slice, i, j)
self.saveViewTile(tile, slice, i, j)
tile = self.readViewTile(pslice, i, j)
self.saveViewTile(tile, pslice, i, j)
scaling *= self.scalingFactor
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment