ReportWriter.py

from scipy.spatial import distance
import numpy as np
from numpy.core.umath_tests import inner1d
import comparative_works
import pandas as pd
import utils_draw_graphs
import os
import multiprocessing as mp
import utils
import MDN_clustering


class ReportWriter:
    def __init__(self,
                 training_batch_handler,
                 validation_batch_handler,
                 test_batch_handler,
                 parameters,
                 report_df):
        """
        training_batch_handler:
        validation_batch_handler:
        test_batch_handler:
            The three data groups to do full hyperparam tuning on a network.
            Other models may need it.
        parameters: The list of all params.
        report_df: the final report generated by the test run. It should only contain d=0 snippets.
        
        """
        report_df.reset_index(inplace=True)
        # self.write_matlab_csv(report_df)
        compares = comparative_works.comparative_works()
        model_df_dict = {}
        # class_dist_df = compares.classifierComboDistributionEstimator(training_batch_handler, validation_batch_handler, test_batch_handler, parameters,
        #                              report_df)
        model_df_dict['CTRA'] = compares.CTRA_model(training_batch_handler, validation_batch_handler, test_batch_handler, parameters,
                                      report_df)
        model_df_dict['CTRV'] = compares.CTRV_model(training_batch_handler, validation_batch_handler, test_batch_handler, parameters,
                                      report_df)
        model_df_dict['CV'] = compares.CV_model(training_batch_handler, validation_batch_handler, test_batch_handler, parameters,
                                      report_df)
        # HMM_errors = compares.HMMGMM(training_batch_handler,validation_batch_handler,test_batch_handler,parameters,report_df)
        # VGMM is CATEGORICAL!
        # VGMM_df = compares.VGMM(training_batch_handler, validation_batch_handler, test_batch_handler,
        #                                         parameters, report_df)
        model_df_dict['GP'] = compares.GaussianProcesses(training_batch_handler, validation_batch_handler, test_batch_handler,
                                                 parameters, report_df)

        try:
            cluster_mix_weight_threshold = parameters['cluster_mix_weight_threshold']
        except KeyError:
            cluster_mix_weight_threshold = 0.5
        try:
            cluster_eps = float(parameters['cluster_eps'])
        except KeyError:
            cluster_eps = 1.0
        try:
            cluster_min_samples = parameters['cluster_min_samples']
        except KeyError:
            cluster_min_samples = 1
            

        # Cluster all the RNN track outputs
        path_MDN_clusters_arr = []
        path_centroids_arr = []
        path_weights_arr = []
        for track_idx in report_df.track_idx:
            path_MDN_clusters, path_centroids, path_weights = MDN_clustering.cluster_MDN_into_sets(
                report_df[report_df.track_idx == track_idx].mixtures.iloc[0],
                mix_weight_threshold=cluster_mix_weight_threshold,
                eps=cluster_eps, min_samples=cluster_min_samples)
            path_MDN_clusters_arr.append(path_MDN_clusters)
            path_centroids_arr.append(path_centroids)
            path_weights_arr.append(path_weights)
        report_df = report_df.assign(path_MDN_clusters=path_MDN_clusters_arr)
        report_df = report_df.assign(path_centroids=path_centroids_arr)
        report_df = report_df.assign(path_weights=path_weights_arr)

        model_df_dict['RNN'] = report_df

        self.parameters = parameters

        # Score models based on individual directions
        dest_errors_dict = {}
        for relative_destination in report_df.relative_destination.unique():
            errors_dict = {}
            for model_name, model_df in model_df_dict.iteritems():
                print "Evaluating " + model_name + " for class: " + relative_destination
                if 'RNN' in model_name:
                    multihypothesis_list = ['most_confident', 'best']
                else:
                    multihypothesis_list = ['']
                for multihyp_mode in multihypothesis_list:
                    errors_dict[model_name + '-' + multihyp_mode + '-' + relative_destination] = \
                        self.score_model_on_metric(parameters,
                                                   model_df[model_df.relative_destination == relative_destination],
                                                   multihypothesis=multihyp_mode)
                    ideas = None
            dest_errors_dict[relative_destination] = errors_dict

        # Score models totally
        errors_dict = {}
        relative_destination = 'all'
        for model_name, model_df in model_df_dict.iteritems():
            print "Evaluating " + model_name + " for class: " + relative_destination
            if 'RNN' in model_name:
                multihypothesis_list = ['most_confident', 'best']
            else:
                multihypothesis_list = ['']
            for multihyp_mode in multihypothesis_list:
                errors_dict[model_name + '-' + multihyp_mode + '-' + relative_destination] = \
                    self.score_model_on_metric(parameters,
                                               model_df,
                                               multihypothesis=multihyp_mode)
        dest_errors_dict['all'] = errors_dict

        # Consolidate everything, grouped by direction
        directionally_consolidated_errors_dict = {}
        for direction, direction_df in dest_errors_dict.iteritems():
            methodically_consolidated_errors_dict = {}
            for name, df in direction_df.iteritems():
                methodically_consolidated_errors_dict[name] = self._consolidate_errors(df)
            directionally_consolidated_errors_dict[direction] = methodically_consolidated_errors_dict

        # I needs to guarantee ordering of the index for aligning prediction tracks.
        assert (model_df_dict['CTRA'].track_idx == report_df.track_idx).all()
        # Also asser that every track is unique
        assert len(report_df) == len(report_df.track_idx.unique())

        plot_dir = os.path.join(parameters['master_dir'], 'test_data_plots')
        if not os.path.exists(plot_dir):
            os.makedirs(plot_dir)

        print "Reportwriter now plotting test tracks"

        plt_size = (10, 10)

        multithread = True
        if multithread:
            pool = mp.Pool(processes=7, maxtasksperchild=1)
            args = []
            for track_idx in report_df.track_idx:
                #continue
                model_predictions = {}
                for model_name, model_df in model_df_dict.iteritems():
                    model_predictions[model_name] = model_df[model_df.track_idx == track_idx].outputs.iloc[0]

                path_MDN_clusters, path_centroids, path_weights = MDN_clustering.cluster_MDN_into_sets(
                    report_df[report_df.track_idx == track_idx].mixtures.iloc[0],
                    mix_weight_threshold=cluster_mix_weight_threshold, eps=cluster_eps, min_samples=cluster_min_samples)
                for centroid_idx in range(len(path_centroids)):
                    model_predictions['multipath_' + str(centroid_idx)] = np.array(path_centroids[centroid_idx])

                for padding_mask in ['Network']: # :'['None', 'GT', 'Network']:
                    args.append([report_df[report_df.track_idx == track_idx].encoder_sample.iloc[0],
                                                         model_predictions,
                                                         report_df[report_df.track_idx == track_idx].decoder_sample.iloc[0],  # Ground Truth
                                                         report_df[report_df.track_idx == track_idx].mixtures.iloc[0],
                                                         report_df[report_df.track_idx == track_idx].padding_logits.iloc[0],
                                                         report_df[report_df.track_idx == track_idx].trackwise_padding.iloc[0],
                                                         plt_size,
                                                         False,  # draw_prediction_track,
                                                         plot_dir,  # self.plot_directory,
                                                         "best",  # self.log_file_name,
                                                         False,  # multi_sample,
                                                         0,  # self.get_global_step(),
                                                         track_idx,  # graph_number,
                                                         plot_dir,  # fig_dir,
                                                         report_df[report_df.track_idx == track_idx].csv_name.iloc[0],
                                                         report_df[report_df.track_idx == track_idx].relative_destination.iloc[0],
                                                         utils.sanitize_params_dict(parameters), padding_mask])
            results = pool.map(utils_draw_graphs.multiprocess_helper, args)
        else:
            for track_idx in report_df.track_idx:
                # if track_idx != 623:
                #     print track_idx
                #     continue
                model_predictions = {}
                for model_name, model_df in model_df_dict.iteritems():
                    model_predictions[model_name] = model_df[model_df.track_idx == track_idx].outputs.iloc[0]

                path_MDN_clusters, path_centroids, path_weights = MDN_clustering.cluster_MDN_into_sets(
                    report_df[report_df.track_idx
                              == track_idx].mixtures.iloc[0],
                    mix_weight_threshold=cluster_mix_weight_threshold, eps=cluster_eps, min_samples=cluster_min_samples)
                for centroid_idx in range(len(path_centroids)):
                    model_predictions['multipath_' + str(centroid_idx)] = np.array(path_centroids[centroid_idx])

                for padding_mask in ['Network']: #'['None', 'GT', 'Network']:
                    utils_draw_graphs.draw_png_heatmap_graph(report_df[report_df.track_idx == track_idx].encoder_sample.iloc[0],
                                                             model_predictions,
                                                             report_df[report_df.track_idx == track_idx].decoder_sample.iloc[0],  # Ground Truth
                                                             report_df[report_df.track_idx == track_idx].mixtures.iloc[0],
                                                             report_df[report_df.track_idx == track_idx].padding_logits.iloc[0],
                                                             report_df[report_df.track_idx == track_idx].trackwise_padding.iloc[0],
                                                             plt_size,
                                                             False,  # draw_prediction_track,
                                                             plot_dir,  # self.plot_directory,
                                                             "best",  # self.log_file_name,
                                                             False,  # multi_sample,
                                                             0,  # self.get_global_step(),
                                                             track_idx,  # graph_number,
                                                             plot_dir,  # fig_dir,
                                                             report_df[report_df.track_idx == track_idx].csv_name.iloc[0],
                                                             report_df[
                                                                 report_df.track_idx == track_idx].relative_destination.iloc[
                                                                 0],
                                                             parameters,
                                                             padding_mask=padding_mask)


        self.errors_df_dict = directionally_consolidated_errors_dict

        return

    def write_matlab_csv(self,report_df):
        for idx, row in report_df.iterrows():

            ideas = None

    def get_results(self):
        return self.errors_df_dict

    def _consolidate_errors(self, error_df):
        metrics = list(error_df.keys())
        summarized_metrics = {}
        for metric in metrics:
            errors = error_df[metric]
            summarized_metrics[metric + " " + 'median'] = np.median(errors)
            summarized_metrics[metric + " " + 'mean'] = np.mean(errors)
            summarized_metrics[metric + " " + 'worst 5%'] = np.percentile(errors, 95)
            summarized_metrics[metric + " " + 'worst 1%'] = np.percentile(errors, 99)
            if 'horizon' in metric:
                # This is a measure of whether the RNN covered this particular ground truth
                # as defined as an average error of less than 1 meter. This is an aggressive measure, I'd prefer
                # a measure on MHD as it is less senstive to time, only location.
                summarized_metrics[metric + " hit_rate_1m"] = float(len(np.where(np.array(errors) < 1))) / len(errors)
        return summarized_metrics

    # Here, there are many options
    # A) metric variance. LCSS, Hausdorff, etc
    # B) Statistical variance:
        # best mean
        # best worst 5% / 1% / 0.1% <-- It took me ages to get data for a reasonable 0.1% fit!
    @staticmethod
    def score_model_on_metric(parameters, report_df, multihypothesis=''):
        #scores_list = []
        track_scores = {}
        horizon_list = [30, 70]  # 95th percentile lengths: left 55, straight 91, right 208
        horizon_list = [int(h / parameters['subsample']) for h in horizon_list]

        for track in report_df.iterrows():
            pathwise_track_scores = {}
            track = track[1]

            if 'best' in multihypothesis:
                multi_track = track.path_centroids
            elif 'most_confident' in multihypothesis:
                multi_track = track.path_centroids
            else:
                multi_track = [track.outputs]

            for path in multi_track:
                path = np.array(path)

                # Left in for multi-sample compatibility, just take the first answer.
                if len(path.shape) == 3:
                    path = path[0]

                # TRACK ALIGNMENT
                # if the track is shorter than the decoder length, pad it with its last values.
                # It is then trimmed to the length of ground truth, as described by trackwise_padding
                if path.shape[0] < len(track.trackwise_padding):
                    path = np.pad(path, [[len(track.trackwise_padding) - path.shape[0], 0], [0, 0]], mode='edge')
                preds = path[np.logical_not(track.trackwise_padding)]

                gts = track.decoder_sample[np.logical_not(track.trackwise_padding)]

                ### EUCLIDEAN ERROR -- Average
                euclid_error = []
                for pred, gt in zip(preds[:,0:2], gts[:,0:2]):
                    # Iterates over each time-step
                    euclid_error.append(distance.euclidean(pred, gt))
                ### /EUCLIDEAN

                ### HORIZON METRICS
                for dist in horizon_list:
                    if dist >= len(preds):
                        continue
                    horizon_euclid_error = euclid_error[dist]  # distance.euclidean(preds[dist, 0:2], gts[dist,0:2])
                    try:
                        pathwise_track_scores["horizon_steps_" + str(dist)].append(horizon_euclid_error)
                    except KeyError:
                        pathwise_track_scores["horizon_steps_" + str(dist)] = [horizon_euclid_error]

                # Now horizon_dict is keyed by timestep, and contains lists of distance errors
                # Mean, Median, 5% etc can now be done on those arrays.

                ### MODIFIED HAUSDORFF DISTANCE
                # Pulled shamelessly from https://github.com/sapphire008/Python/blob/master/generic/HausdorffDistance.py
                # Thanks sapphire008!

                (A, B) = (preds[:, 0:2], gts[:, 0:2])

                # Find pairwise distance
                # Very occasionally due to rounding errors it D_mat can be a small neg num, resulting in NaN
                D_mat = np.nan_to_num(np.sqrt(inner1d(A, A)[np.newaxis].T +
                                inner1d(B, B) - 2 * (np.dot(A, B.T))))
                # Calculating the forward HD: mean(min(each col))
                try:
                    FHD = np.mean(np.min(D_mat, axis=1))
                # Calculating the reverse HD: mean(min(each row))
                    RHD = np.mean(np.min(D_mat, axis=0))
                    # Calculating mhd
                    MHD = np.max(np.array([FHD, RHD]))
                except:
                    MHD=999999 # Sometimes the test data doesnt contain any of this particular class.
                            # Should not happen in prod
                ### /MHD

                try:
                    pathwise_track_scores['euclidean'].append(np.mean(np.array(euclid_error)))
                    pathwise_track_scores['MHD'].append(MHD)
                except KeyError:
                    pathwise_track_scores['euclidean'] = [np.mean(np.array(euclid_error))]
                    pathwise_track_scores['MHD'] = [MHD]

            #scores_list.append(track_scores)

            # Resolve track scores from pathwise track scores based on `best' or most confident
            if 'most_confident' in multihypothesis:
                for key, value in pathwise_track_scores.iteritems():
                    pathwise_track_scores[key] = pathwise_track_scores[key][np.argmax(track.path_weights)]
            else:
                for key, value in pathwise_track_scores.iteritems():
                    # If it is of length 1, min is the only member, else it returns the 'best'
                    pathwise_track_scores[key] = np.min(pathwise_track_scores[key])

            for key, value in pathwise_track_scores.iteritems():
                try:
                    track_scores[key].append(value)
                except KeyError:
                    track_scores[key] = [value]

            # Now add them to track_scores
        return track_scores

#TODO Make a report_df.pkl for the results, and add a if name is main here to load said cached results.