From 94965f440edb9a788b51a9242d657129ccd554a4 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Thu, 7 Dec 2023 14:24:25 -0500 Subject: [PATCH 01/13] Add image pan_history tracking enpoint for thumbnails --- annotation_tracker/rest.py | 77 +++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index a0081a4..af8ca8f 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -1,10 +1,18 @@ +from bson.objectid import ObjectId from girder.api import access from girder.constants import TokenScope, SortDir -from girder.api.rest import Resource +from girder.api.rest import Resource, setRawResponse, setResponseHeader from girder.api.describe import autoDescribeRoute, Description from .models import Activity +from girder_large_image.models.image_item import ImageItem + +from PIL import Image +import io +import numpy as np +from math import floor, ceil + class AnnotationTrackerResource(Resource): def __init__(self): @@ -13,6 +21,7 @@ def __init__(self): self.route('POST', ('log', ), self.logActivity) self.route('GET', (), self.find) + self.route('GET', ('pan_history', ), self.pan_history) @autoDescribeRoute( Description('Log activity to the database.') @@ -88,3 +97,69 @@ def find(self, sessionId, userId, activity, query, limit, offset, sort): if activity: query['activity'] = activity return Activity().find(query, offset=offset, limit=limit, sort=sort) + + @access.admin(scope=TokenScope.DATA_READ) + @autoDescribeRoute( + Description('Generate image thumbnail displaying HistomicsUI panning history.') + .param('sessionId', 'A session id', required=True) + .param('imageId', 'Image\'s item id', required=True) + .param('maxSize', 'Maximum size of the thumbnail image dimension', + dataType='integer', default=512, required=False) + .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) + .errorResponse() + ) + def pan_history(self, sessionId, imageId, maxSize, limit, offset, sort): + query = { + 'session': sessionId, # sessionId probably isn't unique enough for what we want + 'currentImage': imageId, + 'activity': 'pan', + 'zoom': {'$type': 'int'}, # this can break when window is too large for low zooms, it'll be an arbitrary float + } + events = Activity().find(query, offset=offset, limit=limit, sort=sort) + if events.count() == 0: + return None + + image = ImageItem().findOne({'_id': ObjectId(imageId)}) + item = ImageItem().getMetadata(image) + if not item or 'sizeX' not in item: + return None + + # TODO: getRegion + # get thumbnail + result = ImageItem().getThumbnail(image, checkAndCreate=False, + width=maxSize, height=maxSize, encoding='PNG') + imageData, imageMime = result + + source = Image.open(io.BytesIO(imageData)) + width, height = source.size + scale = width / item['sizeX'] + source.save('../annotation-tracker/image.png') + + # create mask + background_opacity = 0.3 + mask = np.zeros((height, width), dtype=np.uint8) + mask.fill(int(background_opacity * 255)) + + for e in events: + visibleArea = e['visibleArea'] + min_x, max_x = floor(scale * visibleArea['tl']['x']), ceil(scale * visibleArea['br']['x']) + min_y, max_y = floor(scale * visibleArea['tl']['y']), ceil(scale * visibleArea['br']['y']) + + min_x, max_x = max(min_x, 0), min(max_x, width) + min_y, max_y = max(min_y, 0), min(max_y, height) + + mask[min_y:max_y, min_x:max_x] = 255 + + # apply mask + mask_image = Image.fromarray(mask) + mask_image.save('../annotation-tracker/mask.png') + + masked_source = Image.composite(source, mask_image, mask_image) + masked_source.save('../annotation-tracker/masked.png') + + masked_bytes = io.BytesIO() + masked_source.save(masked_bytes, format='PNG') + + setResponseHeader('Content-Type', 'image/png') + setRawResponse() + return masked_bytes.getvalue() From dd73cbeb60795400355e3ff42eaa2603deda0a91 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Wed, 20 Dec 2023 11:55:49 -0500 Subject: [PATCH 02/13] Add region of interest data endpoint "pan_history_json" --- annotation_tracker/rest.py | 143 ++++++++++++++++++++++++------------- 1 file changed, 93 insertions(+), 50 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index af8ca8f..2aeae3f 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -7,6 +7,7 @@ from .models import Activity from girder_large_image.models.image_item import ImageItem +import large_image from PIL import Image import io @@ -22,6 +23,7 @@ def __init__(self): self.route('POST', ('log', ), self.logActivity) self.route('GET', (), self.find) self.route('GET', ('pan_history', ), self.pan_history) + self.route('GET', ('pan_history_json', ), self.pan_history_json) @autoDescribeRoute( Description('Log activity to the database.') @@ -98,68 +100,109 @@ def find(self, sessionId, userId, activity, query, limit, offset, sort): query['activity'] = activity return Activity().find(query, offset=offset, limit=limit, sort=sort) + def activity_rois(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): + """Get a list of pan events for a given image and time range.""" + query = { + 'currentImage': imageId, + 'epochms': {'$gte': startTime, '$lte': endTime}, + 'activity': 'pan', + } + events = Activity().find(query, offset=offset, limit=limit, sort=sort, fields=['epochms', 'visibleArea', 'zoom']) + if events.count() == 0: + return None + + # filter out events that are not close to an integer zoom + def zoom_threshold(zoom): + return abs(zoom - round(zoom)) < zoomThreshold + events = [e for e in events if zoom_threshold(e['zoom'])] + + return events + @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Generate image thumbnail displaying HistomicsUI panning history.') - .param('sessionId', 'A session id', required=True) .param('imageId', 'Image\'s item id', required=True) - .param('maxSize', 'Maximum size of the thumbnail image dimension', - dataType='integer', default=512, required=False) + .param('startTime', 'Start of timeframe to examine (epochms)', dataType='integer', required=True) + .param('endTime', 'End of timeframe to examine (epochms)', dataType='integer', required=True) + .param('zoomThreshold', 'Maximum distance zoom variable can be from an interger', dataType='float', default='0.001', required=False) .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) .errorResponse() ) - def pan_history(self, sessionId, imageId, maxSize, limit, offset, sort): - query = { - 'session': sessionId, # sessionId probably isn't unique enough for what we want - 'currentImage': imageId, - 'activity': 'pan', - 'zoom': {'$type': 'int'}, # this can break when window is too large for low zooms, it'll be an arbitrary float - } - events = Activity().find(query, offset=offset, limit=limit, sort=sort) - if events.count() == 0: + def pan_history(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): + events = self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) + if not events: return None image = ImageItem().findOne({'_id': ObjectId(imageId)}) - item = ImageItem().getMetadata(image) - if not item or 'sizeX' not in item: + meta = ImageItem().getMetadata(image) + if not meta or 'sizeX' not in meta: return None - # TODO: getRegion - # get thumbnail - result = ImageItem().getThumbnail(image, checkAndCreate=False, - width=maxSize, height=maxSize, encoding='PNG') - imageData, imageMime = result - - source = Image.open(io.BytesIO(imageData)) - width, height = source.size - scale = width / item['sizeX'] - source.save('../annotation-tracker/image.png') - - # create mask - background_opacity = 0.3 - mask = np.zeros((height, width), dtype=np.uint8) - mask.fill(int(background_opacity * 255)) + dest = large_image.new() + scales = {} for e in events: - visibleArea = e['visibleArea'] - min_x, max_x = floor(scale * visibleArea['tl']['x']), ceil(scale * visibleArea['br']['x']) - min_y, max_y = floor(scale * visibleArea['tl']['y']), ceil(scale * visibleArea['br']['y']) - - min_x, max_x = max(min_x, 0), min(max_x, width) - min_y, max_y = max(min_y, 0), min(max_y, height) - - mask[min_y:max_y, min_x:max_x] = 255 - - # apply mask - mask_image = Image.fromarray(mask) - mask_image.save('../annotation-tracker/mask.png') - - masked_source = Image.composite(source, mask_image, mask_image) - masked_source.save('../annotation-tracker/masked.png') - - masked_bytes = io.BytesIO() - masked_source.save(masked_bytes, format='PNG') - - setResponseHeader('Content-Type', 'image/png') + tl = e['visibleArea']['tl'] + br = e['visibleArea']['br'] + + # assuming that we know the image size and can zero-out negative values + min_x, max_x = max(floor(tl['x']), 0), max(ceil(br['x']), 0) + min_y, max_y = max(floor(tl['y']), 0), max(ceil(br['y']), 0) + + zoom = e['zoom'] + if zoom not in scales: + scales[zoom] = { + 'region': {'left': min_x, 'right': max_x, 'top': min_y, 'bottom': max_y}, + 'mask': np.zeros((meta['sizeY'], meta['sizeX']), dtype='uint8') + } + else: + region = scales[zoom]['region'] + scales[zoom]['region']['left'] = min(min_x, region['left']) + scales[zoom]['region']['right'] = max(max_x, region['right']) + scales[zoom]['region']['top'] = min(min_y, region['top']) + scales[zoom]['region']['bottom'] = max(max_y, region['bottom']) + + # set the mask for this region + scales[zoom]['mask'][min_y:max_y, min_x:max_x] = 1 + + for zoom in sorted(scales.keys()): + item = scales[zoom] + region = item['region'] + + # assuming that the default scaling for the image is 20x (on zoom=4) + effective_mag = (meta.get('magnification', 20) / 2**4) * (2 ** zoom) + + nparray, mime_type = ImageItem().getRegion( + image, + region=region, + resample=Image.Resampling.NEAREST, + scale={'magnification': effective_mag}, + format=large_image.constants.TILE_FORMAT_NUMPY, + ) + + upscale_factor = 2 ** (meta['levels'] - 1 - zoom) + nparray = nparray.repeat(upscale_factor, axis=0).repeat(upscale_factor, axis=1) + + mask = item['mask'][region['top']:region['bottom'], region['left']:region['right']] + mask = mask.copy() # copy to avoid modifying the original in resize + mask.resize(nparray.shape[:2]) + + dest.addTile(nparray, x=region['left'], y=region['top'], mask=mask) + + data, mime = dest.getRegion() + setResponseHeader('Content-Type', mime) setRawResponse() - return masked_bytes.getvalue() + return data + + @access.admin(scope=TokenScope.DATA_READ) + @autoDescribeRoute( + Description('Return JSON list of ROI bounding boxes for given image/timeframe.') + .param('imageId', 'Image\'s item id', required=True) + .param('startTime', 'Start of timeframe to examine (epochms)', dataType='integer', required=True) + .param('endTime', 'End of timeframe to examine (epochms)', dataType='integer', required=True) + .param('zoomThreshold', 'Maximum distance zoom variable can be from an interger', dataType='float', default='0.001', required=False) + .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) + .errorResponse() + ) + def pan_history_json(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): + return self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) From bd15a70c56e4ea225db829047aacfa018963914a Mon Sep 17 00:00:00 2001 From: willdunklin Date: Wed, 20 Dec 2023 12:06:11 -0500 Subject: [PATCH 03/13] tox reformat --- .gitignore | 1 + annotation_tracker/rest.py | 216 ++++++++++++++++++++++++------------- 2 files changed, 140 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index db07d7b..a5b33ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Python files. __pycache__ *.egg-info +.tox # NPM files. node_modules diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 2aeae3f..80b3b22 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -1,35 +1,35 @@ +from math import ceil, floor + +import large_image +import numpy as np from bson.objectid import ObjectId from girder.api import access -from girder.constants import TokenScope, SortDir +from girder.api.describe import Description, autoDescribeRoute from girder.api.rest import Resource, setRawResponse, setResponseHeader -from girder.api.describe import autoDescribeRoute, Description - -from .models import Activity - +from girder.constants import SortDir, TokenScope from girder_large_image.models.image_item import ImageItem -import large_image - from PIL import Image -import io -import numpy as np -from math import floor, ceil + +from .models import Activity class AnnotationTrackerResource(Resource): def __init__(self): super().__init__() - self.resourceName = 'annotation_tracker' + self.resourceName = "annotation_tracker" - self.route('POST', ('log', ), self.logActivity) - self.route('GET', (), self.find) - self.route('GET', ('pan_history', ), self.pan_history) - self.route('GET', ('pan_history_json', ), self.pan_history_json) + self.route("POST", ("log",), self.logActivity) + self.route("GET", (), self.find) + self.route("GET", ("pan_history",), self.pan_history) + self.route("GET", ("pan_history_json",), self.pan_history_json) @autoDescribeRoute( - Description('Log activity to the database.') - .notes('The return value is a dictionary of session identifiers, each ' - 'with a list of sequenceIds that were logged.') - .jsonParam('activityList', 'The key to reimport.', paramType='body') + Description("Log activity to the database.") + .notes( + "The return value is a dictionary of session identifiers, each " + "with a list of sequenceIds that were logged." + ) + .jsonParam("activityList", "The key to reimport.", paramType="body") ) @access.public def logActivity(self, activityList): @@ -72,137 +72,199 @@ def logActivity(self, activityList): saved = Activity().createActivityList(activityList) results = {} for entry in saved: - results.setdefault(entry['session'], set()).add(entry['sequenceId']) + results.setdefault(entry["session"], set()).add(entry["sequenceId"]) for key in list(results.keys()): results[key] = sorted(results[key]) return results @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( - Description('List or search for activities.') - .responseClass('Activity', array=True) - .param('sessionId', 'A session id', required=False) - .param('userId', 'A user id', required=False) - .param('activity', 'An activity string', required=False) - .jsonParam('query', 'Find activities that match this Mongo query.', - required=False, requireObject=True) - .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) + Description("List or search for activities.") + .responseClass("Activity", array=True) + .param("sessionId", "A session id", required=False) + .param("userId", "A user id", required=False) + .param("activity", "An activity string", required=False) + .jsonParam( + "query", + "Find activities that match this Mongo query.", + required=False, + requireObject=True, + ) + .pagingParams(defaultSort="epochms", defaultSortDir=SortDir.DESCENDING) .errorResponse() ) def find(self, sessionId, userId, activity, query, limit, offset, sort): """Get a list of activities with given search parameters.""" query = query or {} if sessionId: - query['session'] = sessionId + query["session"] = sessionId if userId: - query['userId'] = userId + query["userId"] = userId if activity: - query['activity'] = activity + query["activity"] = activity return Activity().find(query, offset=offset, limit=limit, sort=sort) - def activity_rois(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): + def activity_rois( + self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + ): """Get a list of pan events for a given image and time range.""" query = { - 'currentImage': imageId, - 'epochms': {'$gte': startTime, '$lte': endTime}, - 'activity': 'pan', + "currentImage": imageId, + "epochms": {"$gte": startTime, "$lte": endTime}, + "activity": "pan", } - events = Activity().find(query, offset=offset, limit=limit, sort=sort, fields=['epochms', 'visibleArea', 'zoom']) + events = Activity().find( + query, + offset=offset, + limit=limit, + sort=sort, + fields=["epochms", "visibleArea", "zoom"], + ) if events.count() == 0: return None # filter out events that are not close to an integer zoom def zoom_threshold(zoom): return abs(zoom - round(zoom)) < zoomThreshold - events = [e for e in events if zoom_threshold(e['zoom'])] + + events = [e for e in events if zoom_threshold(e["zoom"])] return events @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( - Description('Generate image thumbnail displaying HistomicsUI panning history.') - .param('imageId', 'Image\'s item id', required=True) - .param('startTime', 'Start of timeframe to examine (epochms)', dataType='integer', required=True) - .param('endTime', 'End of timeframe to examine (epochms)', dataType='integer', required=True) - .param('zoomThreshold', 'Maximum distance zoom variable can be from an interger', dataType='float', default='0.001', required=False) - .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) + Description("Generate image thumbnail displaying HistomicsUI panning history.") + .param("imageId", "Image's item id", required=True) + .param( + "startTime", + "Start of timeframe to examine (epochms)", + dataType="integer", + required=True, + ) + .param( + "endTime", + "End of timeframe to examine (epochms)", + dataType="integer", + required=True, + ) + .param( + "zoomThreshold", + "Maximum distance zoom variable can be from an interger", + dataType="float", + default="0.001", + required=False, + ) + .pagingParams(defaultSort="epochms", defaultSortDir=SortDir.DESCENDING) .errorResponse() ) - def pan_history(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): - events = self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) + def pan_history( + self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + ): + events = self.activity_rois( + imageId, startTime, endTime, zoomThreshold, limit, offset, sort + ) if not events: return None - image = ImageItem().findOne({'_id': ObjectId(imageId)}) + image = ImageItem().findOne({"_id": ObjectId(imageId)}) meta = ImageItem().getMetadata(image) - if not meta or 'sizeX' not in meta: + if not meta or "sizeX" not in meta: return None dest = large_image.new() scales = {} for e in events: - tl = e['visibleArea']['tl'] - br = e['visibleArea']['br'] + tl = e["visibleArea"]["tl"] + br = e["visibleArea"]["br"] # assuming that we know the image size and can zero-out negative values - min_x, max_x = max(floor(tl['x']), 0), max(ceil(br['x']), 0) - min_y, max_y = max(floor(tl['y']), 0), max(ceil(br['y']), 0) + min_x, max_x = max(floor(tl["x"]), 0), max(ceil(br["x"]), 0) + min_y, max_y = max(floor(tl["y"]), 0), max(ceil(br["y"]), 0) - zoom = e['zoom'] + zoom = e["zoom"] if zoom not in scales: scales[zoom] = { - 'region': {'left': min_x, 'right': max_x, 'top': min_y, 'bottom': max_y}, - 'mask': np.zeros((meta['sizeY'], meta['sizeX']), dtype='uint8') + "region": { + "left": min_x, + "right": max_x, + "top": min_y, + "bottom": max_y, + }, + "mask": np.zeros((meta["sizeY"], meta["sizeX"]), dtype="uint8"), } else: - region = scales[zoom]['region'] - scales[zoom]['region']['left'] = min(min_x, region['left']) - scales[zoom]['region']['right'] = max(max_x, region['right']) - scales[zoom]['region']['top'] = min(min_y, region['top']) - scales[zoom]['region']['bottom'] = max(max_y, region['bottom']) + region = scales[zoom]["region"] + scales[zoom]["region"]["left"] = min(min_x, region["left"]) + scales[zoom]["region"]["right"] = max(max_x, region["right"]) + scales[zoom]["region"]["top"] = min(min_y, region["top"]) + scales[zoom]["region"]["bottom"] = max(max_y, region["bottom"]) # set the mask for this region - scales[zoom]['mask'][min_y:max_y, min_x:max_x] = 1 + scales[zoom]["mask"][min_y:max_y, min_x:max_x] = 1 for zoom in sorted(scales.keys()): item = scales[zoom] - region = item['region'] + region = item["region"] # assuming that the default scaling for the image is 20x (on zoom=4) - effective_mag = (meta.get('magnification', 20) / 2**4) * (2 ** zoom) + effective_mag = (meta.get("magnification", 20) / 2**4) * (2**zoom) nparray, mime_type = ImageItem().getRegion( image, region=region, resample=Image.Resampling.NEAREST, - scale={'magnification': effective_mag}, + scale={"magnification": effective_mag}, format=large_image.constants.TILE_FORMAT_NUMPY, ) - upscale_factor = 2 ** (meta['levels'] - 1 - zoom) - nparray = nparray.repeat(upscale_factor, axis=0).repeat(upscale_factor, axis=1) + upscale_factor = 2 ** (meta["levels"] - 1 - zoom) + nparray = nparray.repeat(upscale_factor, axis=0).repeat( + upscale_factor, axis=1 + ) - mask = item['mask'][region['top']:region['bottom'], region['left']:region['right']] - mask = mask.copy() # copy to avoid modifying the original in resize + mask = item["mask"][ + region["top"] : region["bottom"], region["left"] : region["right"] + ] + mask = mask.copy() # copy to avoid modifying the original in resize mask.resize(nparray.shape[:2]) - dest.addTile(nparray, x=region['left'], y=region['top'], mask=mask) + dest.addTile(nparray, x=region["left"], y=region["top"], mask=mask) data, mime = dest.getRegion() - setResponseHeader('Content-Type', mime) + setResponseHeader("Content-Type", mime) setRawResponse() return data @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( - Description('Return JSON list of ROI bounding boxes for given image/timeframe.') - .param('imageId', 'Image\'s item id', required=True) - .param('startTime', 'Start of timeframe to examine (epochms)', dataType='integer', required=True) - .param('endTime', 'End of timeframe to examine (epochms)', dataType='integer', required=True) - .param('zoomThreshold', 'Maximum distance zoom variable can be from an interger', dataType='float', default='0.001', required=False) - .pagingParams(defaultSort='epochms', defaultSortDir=SortDir.DESCENDING) + Description("Return JSON list of ROI bounding boxes for given image/timeframe.") + .param("imageId", "Image's item id", required=True) + .param( + "startTime", + "Start of timeframe to examine (epochms)", + dataType="integer", + required=True, + ) + .param( + "endTime", + "End of timeframe to examine (epochms)", + dataType="integer", + required=True, + ) + .param( + "zoomThreshold", + "Maximum distance zoom variable can be from an interger", + dataType="float", + default="0.001", + required=False, + ) + .pagingParams(defaultSort="epochms", defaultSortDir=SortDir.DESCENDING) .errorResponse() ) - def pan_history_json(self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort): - return self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) + def pan_history_json( + self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + ): + return self.activity_rois( + imageId, startTime, endTime, zoomThreshold, limit, offset, sort + ) From ff72c4e9c2f129311f4ed7d936c72ca9ca0919ec Mon Sep 17 00:00:00 2001 From: willdunklin Date: Tue, 16 Jan 2024 19:04:06 -0500 Subject: [PATCH 04/13] Add spatial downsampling for pan_history regions of interests --- annotation_tracker/rest.py | 186 +++++++++++++++++++++++++------------ 1 file changed, 128 insertions(+), 58 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 80b3b22..727d833 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -104,6 +104,60 @@ def find(self, sessionId, userId, activity, query, limit, offset, sort): query["activity"] = activity return Activity().find(query, offset=offset, limit=limit, sort=sort) + def overlap_th(self, roi, rois, threshold=0.95): + # assuming roi and rois are uniformly sized + # rois being the list of rois used until this point + + if len(rois) == 0: + return True + + def overlap_area(a, b): + return max(0, min(a['right'], b['right']) - max(a['left'], b['left'])) * \ + max(0, min(a['bottom'], b['bottom']) - max(a['top'], b['top'])) + + # find the most overlapping roi so far + max_overlap = 0 + for r in rois: + area = overlap_area(roi, r) + if area > max_overlap: + max_overlap = area + + roi_area = (roi['left'] - roi['right']) * (roi['top'] - roi['bottom']) + + # if the max overlap proportion is less than the minimum threshold, we can add this roi + return (max_overlap / roi_area) < threshold + + def spatial_downsample(self, events, threshold=0.95): + scales = {} + + for e in events: + tl = e["visibleArea"]["tl"] + br = e["visibleArea"]["br"] + + roi = {'top': tl['y'], 'left': tl['x'], 'bottom': br['y'], 'right': br['x']} + roi = {k: max(0, int(v)) for k, v in roi.items()} # clamp pixels to be greater than 0 + # this is needed for getRegion + roi['epochms'] = e['epochms'] + + zoom = e["zoom"] + # TODO: zooms are still not necessarily the same if they meet the zoom_threshold -> make sure to pool the correct ones together + if zoom not in scales: + scales[zoom] = {'rois': [roi]} + else: + scales[zoom]['rois'].append(roi) + + for zoom, regions in scales.items(): + regions = regions['rois'] + + accepted_regions = [] + for roi in regions: + if self.overlap_th(roi, accepted_regions, threshold=threshold): + accepted_regions.append(roi) + + scales[zoom]['accepted_regions'] = accepted_regions + + return scales + def activity_rois( self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort ): @@ -154,11 +208,18 @@ def zoom_threshold(zoom): default="0.001", required=False, ) + .param( + "areaThreshold", + "Minimum ratio of (rectangle area overlap / rectangle area) before resampling occurs", + dataType="float", + default="0.95", + required=False, + ) .pagingParams(defaultSort="epochms", defaultSortDir=SortDir.DESCENDING) .errorResponse() ) def pan_history( - self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + self, imageId, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort ): events = self.activity_rois( imageId, startTime, endTime, zoomThreshold, limit, offset, sort @@ -171,67 +232,70 @@ def pan_history( if not meta or "sizeX" not in meta: return None - dest = large_image.new() - scales = {} + # collect roi's based on spatial downsampling + scales = self.spatial_downsample(events, threshold=areaThreshold) - for e in events: - tl = e["visibleArea"]["tl"] - br = e["visibleArea"]["br"] + origin = None + composite = large_image.new() - # assuming that we know the image size and can zero-out negative values - min_x, max_x = max(floor(tl["x"]), 0), max(ceil(br["x"]), 0) - min_y, max_y = max(floor(tl["y"]), 0), max(ceil(br["y"]), 0) + for zoom, data in scales.items(): + # assuming that the default scaling for the image is 20x (on zoom==meta["levels"]-1) if otherwise unspecified by image metadata + # TODO: remove this assumption... (meta['magnification'] might always be set?) + upscale_factor = 2 ** (meta["levels"] - 1 - zoom) + effective_mag = meta.get("magnification", 20) / upscale_factor - zoom = e["zoom"] - if zoom not in scales: - scales[zoom] = { - "region": { - "left": min_x, - "right": max_x, - "top": min_y, - "bottom": max_y, - }, - "mask": np.zeros((meta["sizeY"], meta["sizeX"]), dtype="uint8"), + bounds = None + scale_image = large_image.new() + + for region in data['accepted_regions']: + if origin is None: + center_x = (region['left'] + region['right']) // 2 + center_y = (region['top'] + region['bottom']) // 2 + origin = { + 'top': center_y, + 'bottom': center_y, + 'left': center_x, + 'right': center_x, + } + + # get pixel data for roi at given magnification + nparray, mime = ImageItem().getRegion( + image, + region=region, + resample=Image.Resampling.NEAREST, + scale={"magnification": effective_mag}, + format=large_image.constants.TILE_FORMAT_NUMPY, + ) + + # translate region by the origin + translated = {k: region[k] - origin[k] for k in origin.keys()} + + # track the bounds of the scale_image w.r.t. the origin + if bounds is None: + bounds = translated + bounds = { + 'top': min(bounds['top'], translated['top']), + 'left': min(bounds['left'], translated['left']), } - else: - region = scales[zoom]["region"] - scales[zoom]["region"]["left"] = min(min_x, region["left"]) - scales[zoom]["region"]["right"] = max(max_x, region["right"]) - scales[zoom]["region"]["top"] = min(min_y, region["top"]) - scales[zoom]["region"]["bottom"] = max(max_y, region["bottom"]) - - # set the mask for this region - scales[zoom]["mask"][min_y:max_y, min_x:max_x] = 1 - - for zoom in sorted(scales.keys()): - item = scales[zoom] - region = item["region"] - - # assuming that the default scaling for the image is 20x (on zoom=4) - effective_mag = (meta.get("magnification", 20) / 2**4) * (2**zoom) - - nparray, mime_type = ImageItem().getRegion( - image, - region=region, - resample=Image.Resampling.NEAREST, - scale={"magnification": effective_mag}, - format=large_image.constants.TILE_FORMAT_NUMPY, - ) - upscale_factor = 2 ** (meta["levels"] - 1 - zoom) - nparray = nparray.repeat(upscale_factor, axis=0).repeat( - upscale_factor, axis=1 - ) + # translate to scale_image coordinates via the upscale_factor + x = translated['left'] // upscale_factor + y = translated['top'] // upscale_factor + scale_image.addTile(nparray, x=x, y=y) - mask = item["mask"][ - region["top"] : region["bottom"], region["left"] : region["right"] - ] - mask = mask.copy() # copy to avoid modifying the original in resize - mask.resize(nparray.shape[:2]) + # get raw scale_image for composition + nparray, mime = scale_image.getRegion(format=large_image.constants.TILE_FORMAT_NUMPY) - dest.addTile(nparray, x=region["left"], y=region["top"], mask=mask) + # TODO: remove this visualization + if zoom == 3: + nparray //= 4 + nparray *= 3 - data, mime = dest.getRegion() + # upscale scale_image region w.r.t. upscale_factor + nparray = nparray.repeat(upscale_factor, axis=0).repeat(upscale_factor, axis=1) + composite.addTile(nparray, x=bounds['left'], y=bounds['top']) + + data, mime = composite.getRegion(encoding='PNG') setResponseHeader("Content-Type", mime) setRawResponse() return data @@ -259,12 +323,18 @@ def pan_history( default="0.001", required=False, ) + .param( + "areaThreshold", + "Minimum ratio of (rectangle area overlap / rectangle area) before resampling occurs", + dataType="float", + default="0.95", + required=False, + ) .pagingParams(defaultSort="epochms", defaultSortDir=SortDir.DESCENDING) .errorResponse() ) def pan_history_json( - self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + self, imageId, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort ): - return self.activity_rois( - imageId, startTime, endTime, zoomThreshold, limit, offset, sort - ) + events = self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) + return self.spatial_downsample(events, threshold=areaThreshold) From 5e915c987fb820b324f9d2157aa46b82328fb67c Mon Sep 17 00:00:00 2001 From: willdunklin Date: Thu, 25 Jan 2024 11:06:11 -0500 Subject: [PATCH 05/13] Add large_image_source_multi compositing --- annotation_tracker/rest.py | 47 +++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 727d833..a5e2d52 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -1,6 +1,8 @@ -from math import ceil, floor +import json import large_image +import large_image_source_multi + import numpy as np from bson.objectid import ObjectId from girder.api import access @@ -146,7 +148,8 @@ def spatial_downsample(self, events, threshold=0.95): else: scales[zoom]['rois'].append(roi) - for zoom, regions in scales.items(): + # from pprint import pprint + for zoom, regions in sorted(scales.items()): regions = regions['rois'] accepted_regions = [] @@ -236,9 +239,13 @@ def pan_history( scales = self.spatial_downsample(events, threshold=areaThreshold) origin = None - composite = large_image.new() + bounds_list = {} + + for zoom, data in sorted(scales.items()): + # discard roi's with scale larger than limit + if zoom >= meta['levels']: + continue - for zoom, data in scales.items(): # assuming that the default scaling for the image is 20x (on zoom==meta["levels"]-1) if otherwise unspecified by image metadata # TODO: remove this assumption... (meta['magnification'] might always be set?) upscale_factor = 2 ** (meta["levels"] - 1 - zoom) @@ -283,17 +290,31 @@ def pan_history( y = translated['top'] // upscale_factor scale_image.addTile(nparray, x=x, y=y) - # get raw scale_image for composition - nparray, mime = scale_image.getRegion(format=large_image.constants.TILE_FORMAT_NUMPY) + bounds_list[zoom] = bounds + scale_image.write(f'./img_zoom{zoom}.tiff', lossy=False) - # TODO: remove this visualization - if zoom == 3: - nparray //= 4 - nparray *= 3 + # TODO: refactor things in terms of np arrays (for min along axis in this case) + min_x, min_y = bounds['left'], bounds['top'] + for b in bounds_list.values(): + min_x = min(min_x, b['left']) + min_y = min(min_y, b['top']) - # upscale scale_image region w.r.t. upscale_factor - nparray = nparray.repeat(upscale_factor, axis=0).repeat(upscale_factor, axis=1) - composite.addTile(nparray, x=bounds['left'], y=bounds['top']) + # TODO: sometimes files are cached here, might need to refresh + sources = [] + for zoom in sorted(bounds_list.keys()): + upscale_factor = 2 ** (meta["levels"] - 1 - zoom) + bounds = bounds_list[zoom] + sources.append({ + 'path': f'./img_zoom{zoom}.tiff', + 'z': 0, + 'position': { + 'x': bounds["left"] - min_x, + 'y': bounds["top"] - min_y, + 'scale': upscale_factor, + }, + }) + + composite = large_image_source_multi.open(json.dumps({'sources': sources})) data, mime = composite.getRegion(encoding='PNG') setResponseHeader("Content-Type", mime) From 414a864c1605b2b986e4953f336f2a7ff3300794 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Mon, 29 Jan 2024 13:24:22 -0500 Subject: [PATCH 06/13] Upload itermediate/composite files to girder --- annotation_tracker/rest.py | 209 +++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 92 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index a5e2d52..ab7efaf 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -1,14 +1,16 @@ -import json +import os +import tempfile import large_image -import large_image_source_multi - import numpy as np +import yaml from bson.objectid import ObjectId from girder.api import access from girder.api.describe import Description, autoDescribeRoute -from girder.api.rest import Resource, setRawResponse, setResponseHeader +from girder.api.rest import Resource from girder.constants import SortDir, TokenScope +from girder.models.folder import Folder +from girder.models.upload import Upload from girder_large_image.models.image_item import ImageItem from PIL import Image @@ -124,7 +126,7 @@ def overlap_area(a, b): if area > max_overlap: max_overlap = area - roi_area = (roi['left'] - roi['right']) * (roi['top'] - roi['bottom']) + roi_area = (roi["left"] - roi["right"]) * (roi["top"] - roi["bottom"]) # if the max overlap proportion is less than the minimum threshold, we can add this roi return (max_overlap / roi_area) < threshold @@ -136,28 +138,27 @@ def spatial_downsample(self, events, threshold=0.95): tl = e["visibleArea"]["tl"] br = e["visibleArea"]["br"] - roi = {'top': tl['y'], 'left': tl['x'], 'bottom': br['y'], 'right': br['x']} + roi = {"top": tl["y"], "left": tl["x"], "bottom": br["y"], "right": br["x"]} roi = {k: max(0, int(v)) for k, v in roi.items()} # clamp pixels to be greater than 0 - # this is needed for getRegion - roi['epochms'] = e['epochms'] + # this is needed for getRegion + roi["epochms"] = e["epochms"] zoom = e["zoom"] # TODO: zooms are still not necessarily the same if they meet the zoom_threshold -> make sure to pool the correct ones together if zoom not in scales: - scales[zoom] = {'rois': [roi]} + scales[zoom] = {"rois": [roi]} else: - scales[zoom]['rois'].append(roi) + scales[zoom]["rois"].append(roi) - # from pprint import pprint for zoom, regions in sorted(scales.items()): - regions = regions['rois'] + regions = regions["rois"] accepted_regions = [] for roi in regions: if self.overlap_th(roi, accepted_regions, threshold=threshold): accepted_regions.append(roi) - scales[zoom]['accepted_regions'] = accepted_regions + scales[zoom]["accepted_regions"] = accepted_regions return scales @@ -192,6 +193,7 @@ def zoom_threshold(zoom): @autoDescribeRoute( Description("Generate image thumbnail displaying HistomicsUI panning history.") .param("imageId", "Image's item id", required=True) + .param("destFolder", "Destination folder id", required=True) .param( "startTime", "Start of timeframe to examine (epochms)", @@ -222,7 +224,7 @@ def zoom_threshold(zoom): .errorResponse() ) def pan_history( - self, imageId, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort + self, imageId, destFolder, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort ): events = self.activity_rois( imageId, startTime, endTime, zoomThreshold, limit, offset, sort @@ -230,96 +232,119 @@ def pan_history( if not events: return None + user = self.getCurrentUser() + folder = Folder().load(destFolder, user=user) + image = ImageItem().findOne({"_id": ObjectId(imageId)}) meta = ImageItem().getMetadata(image) if not meta or "sizeX" not in meta: return None + image_name = os.path.splitext(image["name"])[0] # filename without extension # collect roi's based on spatial downsampling scales = self.spatial_downsample(events, threshold=areaThreshold) origin = None - bounds_list = {} - - for zoom, data in sorted(scales.items()): - # discard roi's with scale larger than limit - if zoom >= meta['levels']: - continue - - # assuming that the default scaling for the image is 20x (on zoom==meta["levels"]-1) if otherwise unspecified by image metadata - # TODO: remove this assumption... (meta['magnification'] might always be set?) - upscale_factor = 2 ** (meta["levels"] - 1 - zoom) - effective_mag = meta.get("magnification", 20) / upscale_factor - - bounds = None - scale_image = large_image.new() - - for region in data['accepted_regions']: - if origin is None: - center_x = (region['left'] + region['right']) // 2 - center_y = (region['top'] + region['bottom']) // 2 - origin = { - 'top': center_y, - 'bottom': center_y, - 'left': center_x, - 'right': center_x, + min_xys = {} + + with tempfile.TemporaryDirectory() as tempdir: + for zoom, data in sorted(scales.items()): + # discard roi's with scale larger than limit + if zoom >= meta["levels"]: + continue + + # assuming that the default scaling for the image is 20x (on zoom==meta["levels"]-1) + upscale_factor = 2 ** (meta["levels"] - 1 - zoom) + effective_mag = meta.get("magnification", 20) / upscale_factor + + min_xy = None + scale_image = large_image.new() + + for region in data["accepted_regions"]: + if origin is None: + center_x = (region["left"] + region["right"]) // 2 + center_y = (region["top"] + region["bottom"]) // 2 + origin = np.array([center_x, center_y]) + + # get pixel data for roi at given magnification + nparray, mime = ImageItem().getRegion( + image, + region=region, + resample=Image.Resampling.NEAREST, + scale={"magnification": effective_mag}, + format=large_image.constants.TILE_FORMAT_NUMPY, + ) + + region = np.array([region["left"], region["top"]]) + translated = region - origin + + # track the bounds of the scale_image w.r.t. the origin + if min_xy is None: + min_xy = translated + min_xy = np.min([min_xy, translated], axis=0) + + # transform to scale_image coordinates via the upscale_factor + translated //= upscale_factor + scale_image.addTile(nparray, x=translated[0], y=translated[1]) + + min_xys[zoom] = min_xy + + file_name = f"zoom_{zoom}_{image_name}.tiff" + file_path = os.path.join(tempdir, file_name) + scale_image.write(file_path, lossy=False) + + # add the file to girder instance + with open(file_path, "rb") as scale_image_file: + Upload().uploadFromFile( + scale_image_file, + os.path.getsize(file_path), + file_name, + parentType="folder", + parent=folder, + user=user, + ) + + # get the minimum x & y across all scales + global_min_xy = np.min(np.array(list(min_xys.values())), axis=0) + + source_list = [] + for zoom in sorted(min_xys.keys()): + upscale_factor = 2 ** (meta["levels"] - 1 - zoom) + position = min_xys[zoom] - global_min_xy + + source_list.append( + { + "path": f"./zoom_{zoom}_{image_name}.tiff", + "z": 0, + "position": { + "x": int(position[0]), + "y": int(position[1]), + "scale": upscale_factor, + }, + # # TODO: including band styles for multiple sources in the same tile can break compositing + # 'params': {'style': {'bands': [ + # {'palette': '#f00', 'band': 1}, + # {'palette': '#0f0', 'band': 2}, + # {'palette': '#00f', 'band': 3}, + # ]}} } - - # get pixel data for roi at given magnification - nparray, mime = ImageItem().getRegion( - image, - region=region, - resample=Image.Resampling.NEAREST, - scale={"magnification": effective_mag}, - format=large_image.constants.TILE_FORMAT_NUMPY, ) - # translate region by the origin - translated = {k: region[k] - origin[k] for k in origin.keys()} - - # track the bounds of the scale_image w.r.t. the origin - if bounds is None: - bounds = translated - bounds = { - 'top': min(bounds['top'], translated['top']), - 'left': min(bounds['left'], translated['left']), - } - - # translate to scale_image coordinates via the upscale_factor - x = translated['left'] // upscale_factor - y = translated['top'] // upscale_factor - scale_image.addTile(nparray, x=x, y=y) - - bounds_list[zoom] = bounds - scale_image.write(f'./img_zoom{zoom}.tiff', lossy=False) - - # TODO: refactor things in terms of np arrays (for min along axis in this case) - min_x, min_y = bounds['left'], bounds['top'] - for b in bounds_list.values(): - min_x = min(min_x, b['left']) - min_y = min(min_y, b['top']) - - # TODO: sometimes files are cached here, might need to refresh - sources = [] - for zoom in sorted(bounds_list.keys()): - upscale_factor = 2 ** (meta["levels"] - 1 - zoom) - bounds = bounds_list[zoom] - sources.append({ - 'path': f'./img_zoom{zoom}.tiff', - 'z': 0, - 'position': { - 'x': bounds["left"] - min_x, - 'y': bounds["top"] - min_y, - 'scale': upscale_factor, - }, - }) - - composite = large_image_source_multi.open(json.dumps({'sources': sources})) - - data, mime = composite.getRegion(encoding='PNG') - setResponseHeader("Content-Type", mime) - setRawResponse() - return data + file_name = f"composite_{image_name}.yml" + file_path = os.path.join(tempdir, file_name) + + with open(file_path, "w") as yml_file: + yml_file.write(f'---\n{yaml.dump({"sources": source_list})}') + + with open(file_path, "rb") as yml_file: + return Upload().uploadFromFile( + yml_file, + os.path.getsize(file_path), + file_name, + parentType="folder", + parent=folder, + user=user, + ) @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( From 273e673b78b0b176ad208b00300f607ab0c8c56c Mon Sep 17 00:00:00 2001 From: willdunklin Date: Mon, 29 Jan 2024 14:13:50 -0500 Subject: [PATCH 07/13] Add support for approximate zoom levels in spatial_downsample --- annotation_tracker/rest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index ab7efaf..2fd32e5 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -140,11 +140,10 @@ def spatial_downsample(self, events, threshold=0.95): roi = {"top": tl["y"], "left": tl["x"], "bottom": br["y"], "right": br["x"]} roi = {k: max(0, int(v)) for k, v in roi.items()} # clamp pixels to be greater than 0 - # this is needed for getRegion + # this is needed for getRegion roi["epochms"] = e["epochms"] - zoom = e["zoom"] - # TODO: zooms are still not necessarily the same if they meet the zoom_threshold -> make sure to pool the correct ones together + zoom = e["rounded_zoom"] if zoom not in scales: scales[zoom] = {"rois": [roi]} else: @@ -185,7 +184,12 @@ def activity_rois( def zoom_threshold(zoom): return abs(zoom - round(zoom)) < zoomThreshold - events = [e for e in events if zoom_threshold(e["zoom"])] + # filter events based on zoom value proximity & store the rounded zoom value + events = [ + {**e, "rounded_zoom": round(e["zoom"])} + for e in events + if zoom_threshold(e["zoom"]) + ] return events From 2977ae3a6b450b2c0d72ef05b258d5bf1b8da46c Mon Sep 17 00:00:00 2001 From: willdunklin Date: Mon, 29 Jan 2024 14:17:50 -0500 Subject: [PATCH 08/13] Rename zoomThreshold API param to zoomPrecision --- annotation_tracker/rest.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 2fd32e5..9c9afdd 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -162,7 +162,7 @@ def spatial_downsample(self, events, threshold=0.95): return scales def activity_rois( - self, imageId, startTime, endTime, zoomThreshold, limit, offset, sort + self, imageId, startTime, endTime, zoomPrecision, limit, offset, sort ): """Get a list of pan events for a given image and time range.""" query = { @@ -182,7 +182,7 @@ def activity_rois( # filter out events that are not close to an integer zoom def zoom_threshold(zoom): - return abs(zoom - round(zoom)) < zoomThreshold + return abs(zoom - round(zoom)) < zoomPrecision # filter events based on zoom value proximity & store the rounded zoom value events = [ @@ -211,8 +211,8 @@ def zoom_threshold(zoom): required=True, ) .param( - "zoomThreshold", - "Maximum distance zoom variable can be from an interger", + "zoomPrecision", + "Maximum deviance zoom variable can be from a round interger value", dataType="float", default="0.001", required=False, @@ -228,10 +228,10 @@ def zoom_threshold(zoom): .errorResponse() ) def pan_history( - self, imageId, destFolder, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort + self, imageId, destFolder, startTime, endTime, zoomPrecision, areaThreshold, limit, offset, sort ): events = self.activity_rois( - imageId, startTime, endTime, zoomThreshold, limit, offset, sort + imageId, startTime, endTime, zoomPrecision, limit, offset, sort ) if not events: return None @@ -367,7 +367,7 @@ def pan_history( required=True, ) .param( - "zoomThreshold", + "zoomPrecision", "Maximum distance zoom variable can be from an interger", dataType="float", default="0.001", @@ -384,7 +384,7 @@ def pan_history( .errorResponse() ) def pan_history_json( - self, imageId, startTime, endTime, zoomThreshold, areaThreshold, limit, offset, sort + self, imageId, startTime, endTime, zoomPrecision, areaThreshold, limit, offset, sort ): - events = self.activity_rois(imageId, startTime, endTime, zoomThreshold, limit, offset, sort) + events = self.activity_rois(imageId, startTime, endTime, zoomPrecision, limit, offset, sort) return self.spatial_downsample(events, threshold=areaThreshold) From 42bf8aa6f14e28cfb41d3229e2c69fcc70336f24 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Mon, 29 Jan 2024 14:22:13 -0500 Subject: [PATCH 09/13] Update description for areaThreshold API param --- annotation_tracker/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 9c9afdd..e8bf47a 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -219,7 +219,7 @@ def zoom_threshold(zoom): ) .param( "areaThreshold", - "Minimum ratio of (rectangle area overlap / rectangle area) before resampling occurs", + "Used in spatial ROI downsampling. Minimum ratio of (ROI rectangle area overlap / rectangle area) before resample occurs", dataType="float", default="0.95", required=False, From 8f07e5850c6937c64d63649921a1fef161a6b9de Mon Sep 17 00:00:00 2001 From: willdunklin Date: Fri, 9 Feb 2024 11:53:06 -0500 Subject: [PATCH 10/13] Add band compositing params to composite yaml file --- annotation_tracker/rest.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index e8bf47a..01bf218 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -274,7 +274,6 @@ def pan_history( nparray, mime = ImageItem().getRegion( image, region=region, - resample=Image.Resampling.NEAREST, scale={"magnification": effective_mag}, format=large_image.constants.TILE_FORMAT_NUMPY, ) @@ -325,12 +324,16 @@ def pan_history( "y": int(position[1]), "scale": upscale_factor, }, - # # TODO: including band styles for multiple sources in the same tile can break compositing - # 'params': {'style': {'bands': [ - # {'palette': '#f00', 'band': 1}, - # {'palette': '#0f0', 'band': 2}, - # {'palette': '#00f', 'band': 3}, - # ]}} + 'params': {'style': {'bands': [ + {'palette': '#f00', 'band': 1}, + {'palette': '#0f0', 'band': 2}, + {'palette': '#00f', 'band': 3}, + { + 'palette': ['#fff0', '#ffff'], + 'band': 4, + 'composite': 'multiply' + }, + ]}} } ) From 9c7382b9f4b21bb41c64a320167458cc5699396e Mon Sep 17 00:00:00 2001 From: willdunklin Date: Wed, 14 Feb 2024 11:29:13 -0500 Subject: [PATCH 11/13] Refactor ROI data collection separate from pan_history image generation --- annotation_tracker/rest.py | 239 +++++++++++++++++++++++++++---------- 1 file changed, 179 insertions(+), 60 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 01bf218..2c6fc6a 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -12,7 +12,6 @@ from girder.models.folder import Folder from girder.models.upload import Upload from girder_large_image.models.image_item import ImageItem -from PIL import Image from .models import Activity @@ -116,8 +115,8 @@ def overlap_th(self, roi, rois, threshold=0.95): return True def overlap_area(a, b): - return max(0, min(a['right'], b['right']) - max(a['left'], b['left'])) * \ - max(0, min(a['bottom'], b['bottom']) - max(a['top'], b['top'])) + return max(0, min(a["right"], b["right"]) - max(a["left"], b["left"])) * \ + max(0, min(a["bottom"], b["bottom"]) - max(a["top"], b["top"])) # find the most overlapping roi so far max_overlap = 0 @@ -193,6 +192,101 @@ def zoom_threshold(zoom): return events + def extract_rois( + self, + imageId, + startTime, + endTime, + zoomPrecision, + areaThreshold, + limit, + offset, + sort, + ): + events = self.activity_rois( + imageId, startTime, endTime, zoomPrecision, limit, offset, sort + ) + if not events: + return None + + image = ImageItem().findOne({"_id": ObjectId(imageId)}) + meta = ImageItem().getMetadata(image) + if not meta or "sizeX" not in meta: + return None + + # collect roi's based on spatial downsampling + scales = self.spatial_downsample(events, threshold=areaThreshold) + + origin = None + zoom_patches = {} + + for zoom, data in sorted(scales.items()): + # discard roi's with scale larger than limit + if zoom >= meta["levels"]: + continue + + min_xy, max_xy = None, None + rois = [] + + for region in data["accepted_regions"]: + if origin is None: + center_x = (region["left"] + region["right"]) // 2 + center_y = (region["top"] + region["bottom"]) // 2 + origin = np.array([center_x, center_y]) + + region_xy = np.array([region["left"], region["top"]]) + region_wh = np.array( + [region["right"] - region["left"], region["bottom"] - region["top"]] + ) + translated = region_xy - origin + + # track the bounds of the scale_image w.r.t. the origin + if min_xy is None: + min_xy = translated + if max_xy is None: + max_xy = translated + region_wh + + min_xy = np.min([min_xy, translated], axis=0) + max_xy = np.max([max_xy, translated + region_wh], axis=0) + + # store the roi info + rois.append( + { + "left": translated[0], + "top": translated[1], + "width": region_wh[0], + "height": region_wh[1], + # metadata + "zoom": zoom, + "epochms": region["epochms"], + } + ) + + zoom_patches[zoom] = { + "min_xy": min_xy, + "max_xy": max_xy, + # data we care about logging + "left": min_xy[0], + "top": min_xy[1], + "width": max_xy[0] - min_xy[0], + "height": max_xy[1] - min_xy[1], + "rois": rois, + } + + # get the max/min x & y across all levels + global_min = np.min([lvl["min_xy"] for lvl in zoom_patches.values()], axis=0) + global_max = np.max([lvl["max_xy"] for lvl in zoom_patches.values()], axis=0) + + panned_area = { + "left": global_min[0], + "top": global_min[1], + "width": global_max[0] - global_min[0], + "height": global_max[1] - global_min[1], + "origin": origin, + "zoom_patches": zoom_patches, + } + return panned_area + @access.admin(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description("Generate image thumbnail displaying HistomicsUI panning history.") @@ -228,12 +322,28 @@ def zoom_threshold(zoom): .errorResponse() ) def pan_history( - self, imageId, destFolder, startTime, endTime, zoomPrecision, areaThreshold, limit, offset, sort + self, + imageId, + destFolder, + startTime, + endTime, + zoomPrecision, + areaThreshold, + limit, + offset, + sort, ): - events = self.activity_rois( - imageId, startTime, endTime, zoomPrecision, limit, offset, sort + pan_data = self.extract_rois( + imageId, + startTime, + endTime, + zoomPrecision, + areaThreshold, + limit, + offset, + sort, ) - if not events: + if not pan_data: return None user = self.getCurrentUser() @@ -243,55 +353,41 @@ def pan_history( meta = ImageItem().getMetadata(image) if not meta or "sizeX" not in meta: return None - image_name = os.path.splitext(image["name"])[0] # filename without extension + image_name = os.path.splitext(image["name"])[0] - # collect roi's based on spatial downsampling - scales = self.spatial_downsample(events, threshold=areaThreshold) - - origin = None - min_xys = {} + origin = pan_data["origin"] + source_list = [] with tempfile.TemporaryDirectory() as tempdir: - for zoom, data in sorted(scales.items()): + for zoom, lvl_data in sorted(pan_data["zoom_patches"].items()): # discard roi's with scale larger than limit if zoom >= meta["levels"]: continue # assuming that the default scaling for the image is 20x (on zoom==meta["levels"]-1) upscale_factor = 2 ** (meta["levels"] - 1 - zoom) - effective_mag = meta.get("magnification", 20) / upscale_factor + magnification = meta.get("magnification", 20) / upscale_factor - min_xy = None scale_image = large_image.new() - - for region in data["accepted_regions"]: - if origin is None: - center_x = (region["left"] + region["right"]) // 2 - center_y = (region["top"] + region["bottom"]) // 2 - origin = np.array([center_x, center_y]) - + for roi in lvl_data["rois"]: # get pixel data for roi at given magnification nparray, mime = ImageItem().getRegion( image, - region=region, - scale={"magnification": effective_mag}, + region={ + "left": roi["left"] + origin[0], + "top": roi["top"] + origin[1], + "right": roi["left"] + roi["width"] + origin[0], + "bottom": roi["top"] + roi["height"] + origin[1], + }, + scale={"magnification": magnification}, format=large_image.constants.TILE_FORMAT_NUMPY, ) - region = np.array([region["left"], region["top"]]) - translated = region - origin - - # track the bounds of the scale_image w.r.t. the origin - if min_xy is None: - min_xy = translated - min_xy = np.min([min_xy, translated], axis=0) - - # transform to scale_image coordinates via the upscale_factor - translated //= upscale_factor - scale_image.addTile(nparray, x=translated[0], y=translated[1]) - - min_xys[zoom] = min_xy + scaled_x = (roi["left"] - lvl_data["left"]) // upscale_factor + scaled_y = (roi["top"] - lvl_data["top"]) // upscale_factor + scale_image.addTile(nparray, x=scaled_x, y=scaled_y) + # write the scale_image to disk file_name = f"zoom_{zoom}_{image_name}.tiff" file_path = os.path.join(tempdir, file_name) scale_image.write(file_path, lossy=False) @@ -307,31 +403,24 @@ def pan_history( user=user, ) - # get the minimum x & y across all scales - global_min_xy = np.min(np.array(list(min_xys.values())), axis=0) - - source_list = [] - for zoom in sorted(min_xys.keys()): - upscale_factor = 2 ** (meta["levels"] - 1 - zoom) - position = min_xys[zoom] - global_min_xy - + # log the source metadata for multisource output source_list.append( { - "path": f"./zoom_{zoom}_{image_name}.tiff", + "path": f"./{file_name}", "z": 0, "position": { - "x": int(position[0]), - "y": int(position[1]), + "x": int(lvl_data["left"] - pan_data["left"]), + "y": int(lvl_data["top"] - pan_data["top"]), "scale": upscale_factor, }, - 'params': {'style': {'bands': [ - {'palette': '#f00', 'band': 1}, - {'palette': '#0f0', 'band': 2}, - {'palette': '#00f', 'band': 3}, + "params": {"style": {"bands": [ + {"palette": "#f00", "band": 1}, + {"palette": "#0f0", "band": 2}, + {"palette": "#00f", "band": 3}, { - 'palette': ['#fff0', '#ffff'], - 'band': 4, - 'composite': 'multiply' + "palette": ["#fff0", "#ffff"], + "band": 4, + "composite": "multiply" }, ]}} } @@ -341,7 +430,7 @@ def pan_history( file_path = os.path.join(tempdir, file_name) with open(file_path, "w") as yml_file: - yml_file.write(f'---\n{yaml.dump({"sources": source_list})}') + yml_file.write(f"---\n{yaml.dump({"sources": source_list})}") with open(file_path, "rb") as yml_file: return Upload().uploadFromFile( @@ -387,7 +476,37 @@ def pan_history( .errorResponse() ) def pan_history_json( - self, imageId, startTime, endTime, zoomPrecision, areaThreshold, limit, offset, sort + self, + imageId, + startTime, + endTime, + zoomPrecision, + areaThreshold, + limit, + offset, + sort, ): - events = self.activity_rois(imageId, startTime, endTime, zoomPrecision, limit, offset, sort) - return self.spatial_downsample(events, threshold=areaThreshold) + panned_area = self.extract_rois( + imageId, + startTime, + endTime, + zoomPrecision, + areaThreshold, + limit, + offset, + sort, + ) + if not panned_area: + return None + + # remove numpy array metadata from the response + zoom_patches = panned_area["zoom_patches"] + for zoom in zoom_patches.keys(): + del zoom_patches[zoom]["min_xy"] + del zoom_patches[zoom]["max_xy"] + + # convert numpy array to dict + origin = panned_area["origin"] + panned_area["origin"] = {"x": int(origin[0]), "y": int(origin[1])} + + return panned_area From ed75e74dbe13136e3057ca2c4f6930823c658d7c Mon Sep 17 00:00:00 2001 From: willdunklin Date: Wed, 14 Feb 2024 11:38:27 -0500 Subject: [PATCH 12/13] Fix reformat string error --- annotation_tracker/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 2c6fc6a..6074e83 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -430,7 +430,7 @@ def pan_history( file_path = os.path.join(tempdir, file_name) with open(file_path, "w") as yml_file: - yml_file.write(f"---\n{yaml.dump({"sources": source_list})}") + yml_file.write(f"---\n{yaml.dump({'sources': source_list})}") with open(file_path, "rb") as yml_file: return Upload().uploadFromFile( From faed6090380d73de315087744c147ed7c4d59795 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Thu, 15 Feb 2024 12:24:19 -0500 Subject: [PATCH 13/13] Add timestamp to pan_history output file names --- annotation_tracker/rest.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/annotation_tracker/rest.py b/annotation_tracker/rest.py index 6074e83..a3a5c78 100644 --- a/annotation_tracker/rest.py +++ b/annotation_tracker/rest.py @@ -1,5 +1,6 @@ import os import tempfile +import time import large_image import numpy as np @@ -353,7 +354,10 @@ def pan_history( meta = ImageItem().getMetadata(image) if not meta or "sizeX" not in meta: return None + + # for output file names image_name = os.path.splitext(image["name"])[0] + now = time.strftime("%Y%m%d-%H%M%S") origin = pan_data["origin"] source_list = [] @@ -388,7 +392,7 @@ def pan_history( scale_image.addTile(nparray, x=scaled_x, y=scaled_y) # write the scale_image to disk - file_name = f"zoom_{zoom}_{image_name}.tiff" + file_name = f"zoom_{zoom}_{image_name}_{now}.tiff" file_path = os.path.join(tempdir, file_name) scale_image.write(file_path, lossy=False) @@ -426,7 +430,7 @@ def pan_history( } ) - file_name = f"composite_{image_name}.yml" + file_name = f"composite_{image_name}_{now}.yml" file_path = os.path.join(tempdir, file_name) with open(file_path, "w") as yml_file: