diff --git a/monstereo/activity.py b/monstereo/activity.py index a106971..204e35a 100644 --- a/monstereo/activity.py +++ b/monstereo/activity.py @@ -27,12 +27,15 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa """ return flag of alert if social distancing is violated """ + + # A) Check whether people are close together xx = centers[idx][0] zz = centers[idx][1] distances = [math.sqrt((xx - centers[i][0]) ** 2 + (zz - centers[i][1]) ** 2) for i, _ in enumerate(centers)] sorted_idxs = np.argsort(distances) indices = [idx_t for idx_t in sorted_idxs[1:] if distances[idx_t] <= threshold_dist] + # B) Check whether people are looking inwards and whether there are no intrusions # Deterministic if n_samples < 2: for idx_t in indices: @@ -47,8 +50,6 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa dds = torch.tensor(dds).view(-1, 1) stds = torch.tensor(stds).view(-1, 1) # stds_te = get_task_error(dds) # similar results to MonoLoco but lower true positive - # print(f'ML : {float(torch.mean(stds))}\n') - # print(f'Task Error: {float(torch.mean(stds_te))}') laplace_d = torch.cat((dds, stds), dim=1) samples_d = laplace_sampling(laplace_d, n_samples=n_samples) @@ -93,19 +94,20 @@ def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False mu_1 = np.array([centers[idx_t][0] + radius * math.cos(theta1), centers[idx_t][1] - radius * math.sin(theta1)]) o_c = (mu_0 + mu_1) / 2 - # Verify they are looking inwards. + # 1) Verify they are looking inwards. # The distance between mus and the center should be less wrt the original position and the center d_new = np.linalg.norm(mu_0 - mu_1) / 2 if social_distance else np.linalg.norm(mu_0 - mu_1) d_0 = np.linalg.norm(x_0 - o_c) d_1 = np.linalg.norm(x_1 - o_c) - # Verify no intrusion for third parties + # 2) Verify no intrusion for third parties if other_centers.size: other_distances = np.linalg.norm(other_centers - o_c.reshape(1, -1), axis=1) else: other_distances = 100 * np.ones((1, 1)) # Condition verified if no other people # Binary Classification + # if np.min(other_distances) > radius: # Ablation without orientation if d_new <= min(d_0, d_1) and np.min(other_distances) > radius: return True return False diff --git a/monstereo/eval/eval_kitti.py b/monstereo/eval/eval_kitti.py index 9ad2710..98e8a19 100644 --- a/monstereo/eval/eval_kitti.py +++ b/monstereo/eval/eval_kitti.py @@ -25,34 +25,45 @@ class EvalKitti: '27', '29', '31', '49') ALP_THRESHOLDS = ('<0.5m', '<1m', '<2m') OUR_METHODS = ['geometric', 'monoloco', 'monoloco_pp', 'pose', 'reid', 'monstereo'] - METHODS_MONO = ['m3d', 'monopsr', 'monodis', 'smoke'] + METHODS_MONO = ['m3d', 'monopsr', 'smoke', 'monodis'] METHODS_STEREO = ['3dop', 'psf', 'pseudo-lidar', 'e2e', 'oc-stereo'] BASELINES = ['task_error', 'pixel_error'] HEADERS = ('method', '<0.5', '<1m', '<2m', 'easy', 'moderate', 'hard', 'all') CATEGORIES = ('pedestrian',) + methods = OUR_METHODS + METHODS_MONO + METHODS_STEREO - def __init__(self, thresh_iou_monoloco=0.3, thresh_iou_base=0.3, thresh_conf_monoloco=0.2, thresh_conf_base=0.5, - verbose=False): + # Set directories + main_dir = os.path.join('data', 'kitti') + dir_gt = os.path.join(main_dir, 'gt') + path_train = os.path.join('splits', 'kitti_train.txt') + path_val = os.path.join('splits', 'kitti_val.txt') + dir_logs = os.path.join('data', 'logs') + assert os.path.exists(dir_logs), "No directory to save final statistics" + dir_fig = os.path.join('data', 'figures') + assert os.path.exists(dir_logs), "No directory to save figures" - self.main_dir = os.path.join('data', 'kitti') - self.dir_gt = os.path.join(self.main_dir, 'gt') - self.methods = self.OUR_METHODS + self.METHODS_MONO + self.METHODS_STEREO - path_train = os.path.join('splits', 'kitti_train.txt') - path_val = os.path.join('splits', 'kitti_val.txt') - dir_logs = os.path.join('data', 'logs') - assert dir_logs, "No directory to save final statistics" + # Set thresholds to obtain comparable recalls + thresh_iou_monoloco = 0.3 + thresh_iou_base = 0.3 + thresh_conf_monoloco = 0.2 + thresh_conf_base = 0.5 + + def __init__(self, args): + + self.verbose = args.verbose + self.net = args.net + self.save = args.save + self.show = args.show now = datetime.datetime.now() now_time = now.strftime("%Y%m%d-%H%M")[2:] - self.path_results = os.path.join(dir_logs, 'eval-' + now_time + '.json') - self.verbose = verbose + self.path_results = os.path.join(self.dir_logs, 'eval-' + now_time + '.json') - self.dic_thresh_iou = {method: (thresh_iou_monoloco if method in self.OUR_METHODS - else thresh_iou_base) - for method in self.methods} - self.dic_thresh_conf = {method: (thresh_conf_monoloco if method in self.OUR_METHODS - else thresh_conf_base) - for method in self.methods} + # Set thresholds for comparable recalls + self.dic_thresh_iou = {method: (self.thresh_iou_monoloco if method in self.OUR_METHODS else self.thresh_iou_base) + for method in self.methods} + self.dic_thresh_conf = {method: (self.thresh_conf_monoloco if method in self.OUR_METHODS else self.thresh_conf_base) + for method in self.methods} # Set thresholds to obtain comparable recall self.dic_thresh_conf['monopsr'] += 0.4 @@ -63,7 +74,7 @@ class EvalKitti: # Extract validation images for evaluation names_gt = tuple(os.listdir(self.dir_gt)) - _, self.set_val = split_training(names_gt, path_train, path_val) + _, self.set_val = split_training(names_gt, self.path_train, self.path_val) # self.set_val = ('002282.txt', ) @@ -130,12 +141,14 @@ class EvalKitti: print('\n' + self.category.upper() + ':') self.show_statistics() - def printer(self, show, save): - if save or show: - show_results(self.dic_stats, self.CLUSTERS, show=show, save=save) - show_spread(self.dic_stats, self.CLUSTERS, show=show, save=save) - show_box_plot(self.errors, self.CLUSTERS, show=show, save=save) - show_task_error(show=show, save=save) + def printer(self): + if self.save or self.show: + show_results(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save) + show_spread(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save) + if self.net == 'monstero': + show_box_plot(self.errors, self.CLUSTERS, self.dir_fig, show=self.show, save=self.save) + else: + show_task_error(self.dir_fig, show=self.show, save=self.save) def _parse_txts(self, path, method): diff --git a/monstereo/eval/generate_kitti.py b/monstereo/eval/generate_kitti.py index 0d7b1c4..70c3233 100644 --- a/monstereo/eval/generate_kitti.py +++ b/monstereo/eval/generate_kitti.py @@ -240,10 +240,10 @@ def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco', ca if mode == 'monstereo': conf_scale = 0.03 elif mode == 'monoloco_pp': - # conf_scale = 0.033 - conf_scale = 0.035 # nuScenes for having same recall + conf_scale = 0.033 + # conf_scale = 0.035 # nuScenes for having same recall else: - conf_scale = 0.055 + conf_scale = 0.05 conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2)) output_list = [alpha] + uv_box[:-1] + hwl + cam_0 + [ry, conf, bi, epi] diff --git a/monstereo/run.py b/monstereo/run.py index 535cd9c..e6ba121 100644 --- a/monstereo/run.py +++ b/monstereo/run.py @@ -179,9 +179,9 @@ def main(): if args.dataset == 'kitti': from .eval import EvalKitti - kitti_eval = EvalKitti(verbose=args.verbose) + kitti_eval = EvalKitti(args) kitti_eval.run() - kitti_eval.printer(show=args.show, save=args.save) + kitti_eval.printer() elif 'nuscenes' in args.dataset: from .train import Trainer diff --git a/monstereo/visuals/figures.py b/monstereo/visuals/figures.py index bf0bddc..81f4d73 100644 --- a/monstereo/visuals/figures.py +++ b/monstereo/visuals/figures.py @@ -17,21 +17,22 @@ DPI = 200 GRID_WIDTH = 0.5 -def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=False, stereo=True): +def show_results(dic_stats, clusters, net, dir_fig, show=False, save=False): """ Visualize error as function of the distance and compare it with target errors based on human height analyses """ phase = 'test' x_min = 3 - x_max = 42 + # x_max = 42 + x_max = 31 y_min = 0 # y_max = 2.2 - y_max = 3.5 if stereo else 5.2 + y_max = 3.5 if net == 'monstereo' else 2.6 xx = np.linspace(x_min, x_max, 100) - excl_clusters = ['all', 'easy', 'moderate', 'hard'] + excl_clusters = ['all', 'easy', 'moderate', 'hard', '49'] clusters = [clst for clst in clusters if clst not in excl_clusters] - styles = printing_styles(stereo) + styles = printing_styles(net) for idx_style, style in enumerate(styles.items()): plt.figure(idx_style, figsize=FIGSIZE) plt.grid(linewidth=GRID_WIDTH) @@ -51,7 +52,7 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F if method in ('monstereo', 'pseudo-lidar'): for i, x in enumerate(xxs): plt.text(x, errs[i], str(cnts[i]), fontsize=FONTSIZE) - if not stereo: + if net == 'monoloco_pp': plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5) # if stereo: # yy_stereo = get_pixel_error(xx) @@ -62,18 +63,18 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F plt.yticks(fontsize=FONTSIZE) if save: plt.tight_layout() - mode = 'stereo' if stereo else 'mono' - path_fig = os.path.join(dir_out, 'results_' + mode + '.png') + path_fig = os.path.join(dir_fig, 'results_' + net + '.png') plt.savefig(path_fig, dpi=DPI) - print("Figure of results " + mode + " saved in {}".format(path_fig)) + print("Figure of results " + net + " saved in {}".format(path_fig)) if show: plt.show() plt.close('all') -def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=False): +def show_spread(dic_stats, clusters, net, dir_fig, show=False, save=False): """Predicted confidence intervals and task error as a function of ground-truth distance""" + assert net in ('monoloco_pp', 'monstereo'), "network not recognized" phase = 'test' excl_clusters = ['all', 'easy', 'moderate', 'hard'] clusters = [clst for clst in clusters if clst not in excl_clusters] @@ -81,42 +82,42 @@ def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=Fa x_max = 42 y_min = 0 - for method in ('monoloco_pp', 'monstereo'): - plt.figure(2, figsize=FIGSIZE) - xxs = get_distances(clusters) - bbs = np.array([dic_stats[phase][method][key]['std_ale'] for key in clusters[:-1]]) - if method == 'monoloco_pp': - y_max = 5 - color = 'deepskyblue' - epis = np.array([dic_stats[phase][method][key]['std_epi'] for key in clusters[:-1]]) - plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)") - else: - y_max = 3.5 - color = 'b' - plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error') - plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8) - xx = np.linspace(x_min, x_max, 100) - plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4) + plt.figure(2, figsize=FIGSIZE) + xxs = get_distances(clusters) + bbs = np.array([dic_stats[phase][net][key]['std_ale'] for key in clusters[:-1]]) + xx = np.linspace(x_min, x_max, 100) + if net == 'monoloco_pp': + y_max = 5 + color = 'deepskyblue' + epis = np.array([dic_stats[phase][net][key]['std_epi'] for key in clusters[:-1]]) + plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)") + else: + y_max = 3.5 + color = 'b' + plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error') + plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8) + plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4) - plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE) - plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE) - plt.xlim(x_min, x_max) - plt.ylim(y_min, y_max) - plt.grid(linewidth=GRID_WIDTH) - plt.legend(prop={'size': FONTSIZE}) - plt.xticks(fontsize=FONTSIZE) - plt.yticks(fontsize=FONTSIZE) - if save: - plt.tight_layout() - path_fig = os.path.join(dir_out, 'spread_' + method + '.png') - plt.savefig(path_fig, dpi=DPI) - print("Figure of confidence intervals saved in {}".format(path_fig)) - if show: - plt.show() - plt.close('all') + plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE) + plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE) + plt.xlim(x_min, x_max) + plt.ylim(y_min, y_max) + plt.grid(linewidth=GRID_WIDTH) + plt.legend(prop={'size': FONTSIZE}) + plt.xticks(fontsize=FONTSIZE) + plt.yticks(fontsize=FONTSIZE) + + if save: + plt.tight_layout() + path_fig = os.path.join(dir_fig, 'spread_' + net + '.png') + plt.savefig(path_fig, dpi=DPI) + print("Figure of confidence intervals saved in {}".format(path_fig)) + if show: + plt.show() + plt.close('all') -def show_task_error(show, save, dir_out='data/figures'): +def show_task_error(dir_fig, show, save): """Task error figure""" plt.figure(3, figsize=FIGSIZE) xx = np.linspace(0.1, 50, 100) @@ -147,7 +148,7 @@ def show_task_error(show, save, dir_out='data/figures'): plt.xticks(fontsize=FONTSIZE) plt.yticks(fontsize=FONTSIZE) if save: - path_fig = os.path.join(dir_out, 'task_error.png') + path_fig = os.path.join(dir_fig, 'task_error.png') plt.savefig(path_fig, dpi=DPI) print("Figure of task error saved in {}".format(path_fig)) if show: @@ -181,7 +182,7 @@ def show_method(save, dir_out='data/figures'): plt.close('all') -def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save=False): +def show_box_plot(dic_errors, clusters, dir_fig, show=False, save=False): import pandas as pd excl_clusters = ['all', 'easy', 'moderate', 'hard'] clusters = [int(clst) for clst in clusters if clst not in excl_clusters] @@ -205,7 +206,7 @@ def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save plt.ylim(y_min, y_max) if save: - path_fig = os.path.join(dir_out, 'box_plot_' + name + '.png') + path_fig = os.path.join(dir_fig, 'box_plot_' + name + '.png') plt.tight_layout() plt.savefig(path_fig, dpi=DPI) print("Figure of box plot saved in {}".format(path_fig)) @@ -300,8 +301,8 @@ def get_percentile(dist_gmm): # mad_d = np.mean(np.abs(dist_d - mu_d)) -def printing_styles(stereo): - if stereo: +def printing_styles(net): + if net == 'monstereo': style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'], "methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'], "mks": ['s', 'p', 'o', 'v', '*', '^'], @@ -309,11 +310,12 @@ def printing_styles(stereo): "colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'], "lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']} else: - style = {"labels": ['Mono3D', 'Geometric Baseline', 'MonoPSR', '3DOP (stereo)', 'MonoLoco', 'Monoloco++'], - "methods": ['m3d', 'geometric', 'monopsr', '3dop', 'monoloco', 'monoloco_pp'], + style = {"labels": ['Geometric Baseline', 'MonoPSR', 'MonoDIS', '3DOP (stereo)', + 'MonoLoco', 'Monoloco++'], + "methods": ['geometric', 'monopsr', 'monodis', '3dop', 'monoloco', 'monoloco_pp'], "mks": ['*', '^', 'p', '.', 's', 'o', 'o'], "mksizes": [6, 6, 6, 6, 6, 6], "lws": [1.5, 1.5, 1.5, 1.5, 1.5, 2.2], - "colors": ['r', 'purple', 'olive', 'darkorange', 'b', 'darkblue'], + "colors": ['purple', 'olive', 'r', 'darkorange', 'b', 'darkblue'], "lstyles": ['solid', 'solid', 'solid', 'dashdot', 'solid', 'solid', ]} return style