refactor of figures for evaluation

monstereo/activity.py

@@ -27,12 +27,15 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
     """
     return flag of alert if social distancing is violated
     """
+
+    # A) Check whether people are close together
     xx = centers[idx][0]
     zz = centers[idx][1]
     distances = [math.sqrt((xx - centers[i][0]) ** 2 + (zz - centers[i][1]) ** 2) for i, _ in enumerate(centers)]
     sorted_idxs = np.argsort(distances)
     indices = [idx_t for idx_t in sorted_idxs[1:] if distances[idx_t] <= threshold_dist]
 
+    # B) Check whether people are looking inwards and whether there are no intrusions
     # Deterministic
     if n_samples < 2:
         for idx_t in indices:
@@ -47,8 +50,6 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
         dds = torch.tensor(dds).view(-1, 1)
         stds = torch.tensor(stds).view(-1, 1)
         # stds_te = get_task_error(dds)  # similar results to MonoLoco but lower true positive
-        # print(f'ML : {float(torch.mean(stds))}\n')
-        # print(f'Task Error: {float(torch.mean(stds_te))}')
         laplace_d = torch.cat((dds, stds), dim=1)
         samples_d = laplace_sampling(laplace_d, n_samples=n_samples)
 
@@ -93,19 +94,20 @@ def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False
         mu_1 = np.array([centers[idx_t][0] + radius * math.cos(theta1), centers[idx_t][1] - radius * math.sin(theta1)])
         o_c = (mu_0 + mu_1) / 2
 
-        # Verify they are looking inwards.
+        # 1) Verify they are looking inwards.
         # The distance between mus and the center should be less wrt the original position and the center
         d_new = np.linalg.norm(mu_0 - mu_1) / 2 if social_distance else np.linalg.norm(mu_0 - mu_1)
         d_0 = np.linalg.norm(x_0 - o_c)
         d_1 = np.linalg.norm(x_1 - o_c)
 
-        # Verify no intrusion for third parties
+        # 2) Verify no intrusion for third parties
         if other_centers.size:
             other_distances = np.linalg.norm(other_centers - o_c.reshape(1, -1), axis=1)
         else:
             other_distances = 100 * np.ones((1, 1))  # Condition verified if no other people
 
         # Binary Classification
+        # if np.min(other_distances) > radius:  # Ablation without orientation
         if d_new <= min(d_0, d_1) and np.min(other_distances) > radius:
             return True
     return False

monstereo/eval/eval_kitti.py

@@ -25,34 +25,45 @@ class EvalKitti:
                 '27', '29', '31', '49')
     ALP_THRESHOLDS = ('<0.5m', '<1m', '<2m')
     OUR_METHODS = ['geometric', 'monoloco', 'monoloco_pp', 'pose', 'reid', 'monstereo']
-    METHODS_MONO = ['m3d', 'monopsr', 'monodis', 'smoke']
+    METHODS_MONO = ['m3d', 'monopsr', 'smoke', 'monodis']
     METHODS_STEREO = ['3dop', 'psf', 'pseudo-lidar', 'e2e', 'oc-stereo']
     BASELINES = ['task_error', 'pixel_error']
     HEADERS = ('method', '<0.5', '<1m', '<2m', 'easy', 'moderate', 'hard', 'all')
     CATEGORIES = ('pedestrian',)
+    methods = OUR_METHODS + METHODS_MONO + METHODS_STEREO
 
-    def __init__(self, thresh_iou_monoloco=0.3, thresh_iou_base=0.3, thresh_conf_monoloco=0.2, thresh_conf_base=0.5,
+    # Set directories
-                 verbose=False):
+    main_dir = os.path.join('data', 'kitti')
+    dir_gt = os.path.join(main_dir, 'gt')
+    path_train = os.path.join('splits', 'kitti_train.txt')
+    path_val = os.path.join('splits', 'kitti_val.txt')
+    dir_logs = os.path.join('data', 'logs')
+    assert os.path.exists(dir_logs), "No directory to save final statistics"
+    dir_fig = os.path.join('data', 'figures')
+    assert os.path.exists(dir_logs), "No directory to save figures"
 
-        self.main_dir = os.path.join('data', 'kitti')
+    # Set thresholds to obtain comparable recalls
-        self.dir_gt = os.path.join(self.main_dir, 'gt')
+    thresh_iou_monoloco = 0.3
-        self.methods = self.OUR_METHODS + self.METHODS_MONO + self.METHODS_STEREO
+    thresh_iou_base = 0.3
-        path_train = os.path.join('splits', 'kitti_train.txt')
+    thresh_conf_monoloco = 0.2
-        path_val = os.path.join('splits', 'kitti_val.txt')
+    thresh_conf_base = 0.5
-        dir_logs = os.path.join('data', 'logs')
+
-        assert dir_logs, "No directory to save final statistics"
+    def __init__(self, args):
+
+        self.verbose = args.verbose
+        self.net = args.net
+        self.save = args.save
+        self.show = args.show
 
         now = datetime.datetime.now()
         now_time = now.strftime("%Y%m%d-%H%M")[2:]
-        self.path_results = os.path.join(dir_logs, 'eval-' + now_time + '.json')
+        self.path_results = os.path.join(self.dir_logs, 'eval-' + now_time + '.json')
-        self.verbose = verbose
 
-        self.dic_thresh_iou = {method: (thresh_iou_monoloco if method in self.OUR_METHODS
+        # Set thresholds for comparable recalls
-                                        else thresh_iou_base)
+        self.dic_thresh_iou = {method: (self.thresh_iou_monoloco if method in self.OUR_METHODS else self.thresh_iou_base)
-                               for method in self.methods}
+                          for method in self.methods}
-        self.dic_thresh_conf = {method: (thresh_conf_monoloco if method in self.OUR_METHODS
+        self.dic_thresh_conf = {method: (self.thresh_conf_monoloco if method in self.OUR_METHODS else self.thresh_conf_base)
-                                         else thresh_conf_base)
+                           for method in self.methods}
-                                for method in self.methods}
 
         # Set thresholds to obtain comparable recall
         self.dic_thresh_conf['monopsr'] += 0.4
@@ -63,7 +74,7 @@ class EvalKitti:
 
         # Extract validation images for evaluation
         names_gt = tuple(os.listdir(self.dir_gt))
-        _, self.set_val = split_training(names_gt, path_train, path_val)
+        _, self.set_val = split_training(names_gt, self.path_train, self.path_val)
 
         # self.set_val = ('002282.txt', )
 
@@ -130,12 +141,14 @@ class EvalKitti:
             print('\n' + self.category.upper() + ':')
             self.show_statistics()
 
-    def printer(self, show, save):
+    def printer(self):
-        if save or show:
+        if self.save or self.show:
-            show_results(self.dic_stats, self.CLUSTERS, show=show, save=save)
+            show_results(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
-            show_spread(self.dic_stats, self.CLUSTERS, show=show, save=save)
+            show_spread(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
-            show_box_plot(self.errors, self.CLUSTERS, show=show, save=save)
+            if self.net == 'monstero':
-            show_task_error(show=show, save=save)
+                show_box_plot(self.errors, self.CLUSTERS, self.dir_fig, show=self.show, save=self.save)
+            else:
+                show_task_error(self.dir_fig, show=self.show, save=self.save)
 
     def _parse_txts(self, path, method):
 

monstereo/eval/generate_kitti.py

@@ -240,10 +240,10 @@ def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco', ca
             if mode == 'monstereo':
                 conf_scale = 0.03
             elif mode == 'monoloco_pp':
-                # conf_scale = 0.033
+                conf_scale = 0.033
-                conf_scale = 0.035  # nuScenes for having same recall
+                # conf_scale = 0.035  # nuScenes for having same recall
             else:
-                conf_scale = 0.055
+                conf_scale = 0.05
             conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2))
 
             output_list = [alpha] + uv_box[:-1] + hwl + cam_0 + [ry, conf, bi, epi]

monstereo/run.py

@@ -179,9 +179,9 @@ def main():
 
             if args.dataset == 'kitti':
                 from .eval import EvalKitti
-                kitti_eval = EvalKitti(verbose=args.verbose)
+                kitti_eval = EvalKitti(args)
                 kitti_eval.run()
-                kitti_eval.printer(show=args.show, save=args.save)
+                kitti_eval.printer()
 
             elif 'nuscenes' in args.dataset:
                 from .train import Trainer

monstereo/visuals/figures.py

@@ -17,21 +17,22 @@ DPI = 200
 GRID_WIDTH = 0.5
 
 
-def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=False, stereo=True):
+def show_results(dic_stats, clusters, net, dir_fig, show=False, save=False):
     """
     Visualize error as function of the distance and compare it with target errors based on human height analyses
     """
 
     phase = 'test'
     x_min = 3
-    x_max = 42
+    # x_max = 42
+    x_max = 31
     y_min = 0
     # y_max = 2.2
-    y_max = 3.5 if stereo else 5.2
+    y_max = 3.5 if net == 'monstereo' else 2.6
     xx = np.linspace(x_min, x_max, 100)
-    excl_clusters = ['all', 'easy', 'moderate', 'hard']
+    excl_clusters = ['all', 'easy', 'moderate', 'hard', '49']
     clusters = [clst for clst in clusters if clst not in excl_clusters]
-    styles = printing_styles(stereo)
+    styles = printing_styles(net)
     for idx_style, style in enumerate(styles.items()):
         plt.figure(idx_style, figsize=FIGSIZE)
         plt.grid(linewidth=GRID_WIDTH)
@@ -51,7 +52,7 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F
             if method in ('monstereo', 'pseudo-lidar'):
                 for i, x in enumerate(xxs):
                     plt.text(x, errs[i], str(cnts[i]), fontsize=FONTSIZE)
-    if not stereo:
+    if net == 'monoloco_pp':
         plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5)
     # if stereo:
     #     yy_stereo = get_pixel_error(xx)
@@ -62,18 +63,18 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F
     plt.yticks(fontsize=FONTSIZE)
     if save:
         plt.tight_layout()
-        mode = 'stereo' if stereo else 'mono'
+        path_fig = os.path.join(dir_fig, 'results_' + net + '.png')
-        path_fig = os.path.join(dir_out, 'results_' + mode + '.png')
         plt.savefig(path_fig, dpi=DPI)
-        print("Figure of results " + mode + " saved in {}".format(path_fig))
+        print("Figure of results " + net + " saved in {}".format(path_fig))
     if show:
         plt.show()
     plt.close('all')
 
 
-def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=False):
+def show_spread(dic_stats, clusters, net, dir_fig, show=False, save=False):
     """Predicted confidence intervals and task error as a function of ground-truth distance"""
 
+    assert net in ('monoloco_pp', 'monstereo'), "network not recognized"
     phase = 'test'
     excl_clusters = ['all', 'easy', 'moderate', 'hard']
     clusters = [clst for clst in clusters if clst not in excl_clusters]
@@ -81,42 +82,42 @@ def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=Fa
     x_max = 42
     y_min = 0
 
-    for method in ('monoloco_pp', 'monstereo'):
+    plt.figure(2, figsize=FIGSIZE)
-        plt.figure(2, figsize=FIGSIZE)
+    xxs = get_distances(clusters)
-        xxs = get_distances(clusters)
+    bbs = np.array([dic_stats[phase][net][key]['std_ale'] for key in clusters[:-1]])
-        bbs = np.array([dic_stats[phase][method][key]['std_ale'] for key in clusters[:-1]])
+    xx = np.linspace(x_min, x_max, 100)
-        if method == 'monoloco_pp':
+    if net == 'monoloco_pp':
-            y_max = 5
+        y_max = 5
-            color = 'deepskyblue'
+        color = 'deepskyblue'
-            epis = np.array([dic_stats[phase][method][key]['std_epi'] for key in clusters[:-1]])
+        epis = np.array([dic_stats[phase][net][key]['std_epi'] for key in clusters[:-1]])
-            plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)")
+        plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)")
-        else:
+    else:
-            y_max = 3.5
+        y_max = 3.5
-            color = 'b'
+        color = 'b'
-            plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
+        plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
-        plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
+    plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
-        xx = np.linspace(x_min, x_max, 100)
+    plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
-        plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
 
-        plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
+    plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
-        plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
+    plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
-        plt.xlim(x_min, x_max)
+    plt.xlim(x_min, x_max)
-        plt.ylim(y_min, y_max)
+    plt.ylim(y_min, y_max)
-        plt.grid(linewidth=GRID_WIDTH)
+    plt.grid(linewidth=GRID_WIDTH)
-        plt.legend(prop={'size': FONTSIZE})
+    plt.legend(prop={'size': FONTSIZE})
-        plt.xticks(fontsize=FONTSIZE)
+    plt.xticks(fontsize=FONTSIZE)
-        plt.yticks(fontsize=FONTSIZE)
+    plt.yticks(fontsize=FONTSIZE)
-        if save:
+
-            plt.tight_layout()
+    if save:
-            path_fig = os.path.join(dir_out, 'spread_' + method + '.png')
+        plt.tight_layout()
-            plt.savefig(path_fig, dpi=DPI)
+        path_fig = os.path.join(dir_fig, 'spread_' + net + '.png')
-            print("Figure of confidence intervals saved in {}".format(path_fig))
+        plt.savefig(path_fig, dpi=DPI)
-        if show:
+        print("Figure of confidence intervals saved in {}".format(path_fig))
-            plt.show()
+    if show:
-        plt.close('all')
+        plt.show()
+    plt.close('all')
 
 
-def show_task_error(show, save, dir_out='data/figures'):
+def show_task_error(dir_fig, show, save):
     """Task error figure"""
     plt.figure(3, figsize=FIGSIZE)
     xx = np.linspace(0.1, 50, 100)
@@ -147,7 +148,7 @@ def show_task_error(show, save, dir_out='data/figures'):
     plt.xticks(fontsize=FONTSIZE)
     plt.yticks(fontsize=FONTSIZE)
     if save:
-        path_fig = os.path.join(dir_out, 'task_error.png')
+        path_fig = os.path.join(dir_fig, 'task_error.png')
         plt.savefig(path_fig, dpi=DPI)
         print("Figure of task error saved in {}".format(path_fig))
     if show:
@@ -181,7 +182,7 @@ def show_method(save, dir_out='data/figures'):
     plt.close('all')
 
 
-def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save=False):
+def show_box_plot(dic_errors, clusters, dir_fig, show=False, save=False):
     import pandas as pd
     excl_clusters = ['all', 'easy', 'moderate', 'hard']
     clusters = [int(clst) for clst in clusters if clst not in excl_clusters]
@@ -205,7 +206,7 @@ def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save
         plt.ylim(y_min, y_max)
 
         if save:
-            path_fig = os.path.join(dir_out, 'box_plot_' + name + '.png')
+            path_fig = os.path.join(dir_fig, 'box_plot_' + name + '.png')
             plt.tight_layout()
             plt.savefig(path_fig, dpi=DPI)
             print("Figure of box plot saved in {}".format(path_fig))
@@ -300,8 +301,8 @@ def get_percentile(dist_gmm):
     # mad_d = np.mean(np.abs(dist_d - mu_d))
 
 
-def printing_styles(stereo):
+def printing_styles(net):
-    if stereo:
+    if net == 'monstereo':
         style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'],
                  "methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'],
                  "mks": ['s', 'p', 'o', 'v', '*', '^'],
@@ -309,11 +310,12 @@ def printing_styles(stereo):
                  "colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'],
                  "lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']}
     else:
-        style = {"labels": ['Mono3D', 'Geometric Baseline', 'MonoPSR', '3DOP (stereo)', 'MonoLoco', 'Monoloco++'],
+        style = {"labels": ['Geometric Baseline', 'MonoPSR', 'MonoDIS', '3DOP (stereo)',
-                 "methods": ['m3d', 'geometric', 'monopsr', '3dop', 'monoloco', 'monoloco_pp'],
+                            'MonoLoco', 'Monoloco++'],
+                 "methods": ['geometric', 'monopsr', 'monodis', '3dop', 'monoloco', 'monoloco_pp'],
                  "mks": ['*', '^', 'p', '.', 's', 'o', 'o'],
                  "mksizes": [6, 6, 6, 6, 6, 6], "lws": [1.5, 1.5, 1.5, 1.5, 1.5, 2.2],
-                 "colors": ['r', 'purple', 'olive', 'darkorange', 'b', 'darkblue'],
+                 "colors": ['purple', 'olive', 'r', 'darkorange', 'b', 'darkblue'],
                  "lstyles": ['solid', 'solid', 'solid', 'dashdot', 'solid', 'solid', ]}
 
     return style