optimized lcc for memory efficiency

Author: sparkvilla

tmlib/workflow/dependencies.py

@@ -141,6 +141,10 @@ class CanonicalWorkflowDependencies(WorkflowDependencies):
         'imextract': {
             'metaconfig'
         },
+
+        'jterator':{
+
+        },
         'popcon' : {
             'jterator'
     }

tmlib/workflow/popcon/api.py

@@ -98,7 +98,7 @@ def delete_previous_job_output(self):
         that were generated by a prior run of the same pipeline as well as all
         children instances for the processed experiment.
         '''
-        pass     
+        pass
 
     def run_job(self, batch, assume_clean_state):
         '''Runs the pipeline, i.e. executes modules sequentially. After
@@ -117,32 +117,34 @@ def run_job(self, batch, assume_clean_state):
                  filter_by(well_id=well_id).all()
                 wellY = sites[0][1]*len(set([i[3] for i in sites]))
                 wellX = sites[0][2]*len(set([i[4]for i in sites]))
-                
-                 
+
+
                 extract_mapobject_type_id = session.query(tm.MapobjectType.id).\
-                filter_by(name=batch['extract_object']).one()[0] 
+                filter_by(name=batch['extract_object']).one()[0]
                 extract_seg_layer_id = session.query(tm.SegmentationLayer.id).\
                 filter_by(mapobject_type_id=extract_mapobject_type_id).one()[0]
                 extract_centroids = session.query(tm.MapobjectSegmentation.geom_centroid,tm.MapobjectSegmentation.mapobject_id,tm.MapobjectSegmentation.label,tm.MapobjectSegmentation.partition_key).\
                 filter_by(segmentation_layer_id=extract_seg_layer_id).all()
                 assign_mapobject_type_id = session.query(tm.MapobjectType.id).\
-                filter_by(name=batch['assign_object']).one()[0] 
+                filter_by(name=batch['assign_object']).one()[0]
                 assign_seg_layer_id = session.query(tm.SegmentationLayer.id).\
                 filter_by(mapobject_type_id=assign_mapobject_type_id).one()[0]
                 assign_centroids = session.query(tm.MapobjectSegmentation.geom_centroid,tm.MapobjectSegmentation.mapobject_id,tm.MapobjectSegmentation.label,tm.MapobjectSegmentation.partition_key).\
                 filter_by(segmentation_layer_id=assign_seg_layer_id).all()
- 
+
                 logger.info('Calculating LCC for well_id %s', well_id)
                 logger.info('Instantiating LCC for extract_object')
                 lcc_extract = LocalCC(extract_centroids, wellY, wellX)
-                logger.info('df lcc_extract: %s',lcc_extract.df.head()) 
+                logger.info('df lcc_extract: %s',lcc_extract.df.head())
                 logger.info(
                   'wellX: %s, wellY: %s, diagonal: %s'
                    , lcc_extract.wellX, lcc_extract.wellY, lcc_extract.well_diagonal)
-                
-                real_lcc = lcc_extract.real_distances()
-                random_lcc = lcc_extract.random_distances() 
+
+                real_lcc = lcc_extract.gen_real_distances()
+                random_lcc = lcc_extract.gen_random_distances()
                 lcc = lcc_extract.get_lcc(real_lcc,random_lcc)
+
+                lcc_extract.df['lcc'] = lcc_extract.df['lcc'].round()
                 logger.info('Instantiating LCC for assign_object')
                 lcc_assign = LocalCC(assign_centroids, wellY, wellX)
 
@@ -153,18 +155,18 @@ def run_job(self, batch, assume_clean_state):
                    , batch['assign_object'], batch['extract_object'])
 
                 #logger.debug(
-                #   'assign: %s extract: %s', lcc_assign.df['mapobject_id'], lcc_extract.df['mapobject_id']) 
-               
-                lcc_extract.df['mapobject_id'] = lcc_assign.df['mapobject_id'] 
+                #   'assign: %s extract: %s', lcc_assign.df['mapobject_id'], lcc_extract.df['mapobject_id'])
+
+                lcc_extract.df['mapobject_id'] = lcc_assign.df['mapobject_id']
 
                 feature_name = 'LocalCellCrowding_{}'.format(batch['extract_object'])
                 feature = session.get_or_create(
-                          tm.Feature, name=feature_name, 
-                          mapobject_type_id=assign_mapobject_type_id,   
-                          is_aggregate=False) 
+                          tm.Feature, name=feature_name,
+                          mapobject_type_id=assign_mapobject_type_id,
+                          is_aggregate=False)
 
 
-                for index, row in lcc_extract.df.iterrows():    
+                for index, row in lcc_extract.df.iterrows():
                     feature_value = session.query(tm.FeatureValues).filter_by(mapobject_id= int(row['mapobject_id']) ).one()
                     session.append_value(feature_value,str(feature.id),row['lcc'].astype(str))
                     session.commit()
@@ -174,4 +176,3 @@ def run_job(self, batch, assume_clean_state):
 
     def collect_job_output(self, batch):
         pass
-

tmlib/workflow/popcon/lcc.py

@@ -12,89 +12,91 @@ class LocalCC(object):
     @staticmethod
     def _get_yx(element):
         ''' Helper. Takes in a WKBElement.
-        Return a shapely.geometry.point.Point object  
+        Return a shapely.geometry.point.Point object
         '''
         return wkb.loads(bytes(element.data))
 
     @staticmethod
-    def _get_df(np_centroids):
-        '''Helper. Takes numpy centroids array
-        Return a sorted dataframe by site and label          
+    def _get_df(centroids):
+        '''Helper. Takes a list of centroids coords.
+        Return a sorted dataframe by site and label
         '''
-        data = np.zeros( (len(np_centroids),5) )
+        data = np.zeros( (len(centroids),5) )
         headers = ['y','x','mapobject_id','label','site']
-        data[:] = np_centroids
+        data[:] = centroids
         data_df = pd.DataFrame(data, columns=headers)
         df = data_df.sort_values(['site', 'label'])
         return df
 
+
+
+
     def __init__(self, centroids, wellY, wellX):
         '''
         Parameters
         ----------
-        centroids: list of tuples (centroid WKB element, mapobject_id, label,site); 
+        centroids: list of tuples (centroid WKB element, mapobject_id, label,site);
         i.e. query objects from MapobjectSegmentation table
         '''
-        self.centroids = centroids
-       
-        self.centroids_coordinates = [(abs(self._get_yx(element[0]).y),abs(self._get_yx(element[0]).x), int(element[1]), int(element[2]), int(element[3])) for element in self.centroids]
-               
+
+        self.centroids_coordinates = [
+              ( round (abs(self._get_yx(element[0]).y),1 ),\
+                round (abs(self._get_yx(element[0]).x),1 ),\
+                int(element[1]),\
+                int(element[2]),\
+                int(element[3]) ) for element in centroids]
+
         self.df = self._get_df(self.centroids_coordinates)
         self.yx_coordinates = np.asarray((self.df['y'],self.df['x'])).transpose()
         self.wellY = wellY
         self.wellX = wellX
         self.well_diagonal = np.round(math.sqrt(self.wellX**2+self.wellY**2))
-        
-       
-   
-    def real_distances(self):
+
+
+    def gen_real_distances(self):
+        '''
+        Returns
+        -------
+        generator
+        Sum of distances calculated from real positions
+        '''
+
+        for yx_real in self.yx_coordinates:
+            real_dist = distance.cdist(np.transpose(yx_real[:,np.newaxis]), self.yx_coordinates, 'euclidean')
+            real_masked = np.ma.masked_where(real_dist==0,real_dist) # mask 0 values
+            real_masked_divide = np.divide(self.well_diagonal, real_masked)
+            yield np.sum(real_masked_divide.filled(fill_value=0))
+
+
+    def gen_random_distances(self):
 	'''
 	Returns
 	-------
-	numpy array of distances from real positions    
-	'''     
-	real_dists = distance.cdist(self.yx_coordinates, self.yx_coordinates, 'euclidean')
-	real_masked = np.ma.masked_where(real_dists==0,real_dists) # mask 0 values 
-	real_masked_divide = np.divide(self.well_diagonal, real_masked)
-	return real_masked_divide.filled(fill_value=0)
-
-    def random_distances(self):
+	generator
+        Sum of distances calculated from random positions
 	'''
-	Returns
-	-------
-	numpy array of distances from random positions    
-	''' 
-	rand_dists= list() 
-	   
+
 	for yx_real in self.yx_coordinates:
 	    y_rand= np.random.uniform(0,self.wellY,len(self.yx_coordinates)-1)
-	    x_rand= np.random.uniform(0,self.wellX,len(self.yx_coordinates)-1) 
+	    x_rand= np.random.uniform(0,self.wellX,len(self.yx_coordinates)-1)
 	    yx_coordinates_random = np.concatenate( (y_rand[:,np.newaxis],x_rand[:,np.newaxis]), axis=1)
-       
+
 	    rand_dist = np.divide(self.well_diagonal, distance.cdist(np.transpose(yx_real[:,np.newaxis]), yx_coordinates_random, 'euclidean') )
-	    rand_dists.append(np.squeeze(rand_dist))
-	return np.asarray(rand_dists)
-        
+	    yield np.sum(np.squeeze(rand_dist))
+
 
     def get_lcc(self,real_dists,random_dists):
 	'''
 	Parameters
 	----------
-	real_dists: numpy arrays of real distances  
-	random_dists: numpy array of random distances
+	real_dists: generator of real distances
+	random_dists: generator of random distances
 
 	Returns
 	-------
-	a numpy array i.e. a LCC value and mapobject_id for centroid     
-	''' 
-	sum_real = np.sum(real_dists, axis=1)
-	sum_random = np.sum(random_dists, axis=1)     
-	lcc = sum_real-sum_random
-	
-        self.df['lcc'] = pd.Series(lcc)   
-	return self.df
-        # lcc[lcc[:,1].argsort()] sort array based on mapobject_id
-	# lcc[lcc[:,0].argsort()] sort array based on lcc value
-	
+	a pandas Dataframe i.e. y,x,mapobject_id,label,site,lcc
+	'''
+	lcc = [re_d-rn_d for re_d,rn_d in zip(real_dists,random_dists)]
 
-    
+        self.df['lcc'] = pd.Series(lcc)
+	return self.df