charles/monoloco
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor of figures for evaluation

Browse Source
This commit is contained in:
Lorenzo 2020-12-09 13:56:59 +01:00
parent 4c5fb0e42c
commit 7beb093a6b
5 changed files with 102 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
monstereo/activity.py
View File

@ -27,12 +27,15 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
"""
return flag of alert if social distancing is violated
"""
# A) Check whether people are close together
xx = centers[idx][0]
zz = centers[idx][1]
distances = [math.sqrt((xx - centers[i][0]) ** 2 + (zz - centers[i][1]) ** 2) for i, _ in enumerate(centers)]
sorted_idxs = np.argsort(distances)
indices = [idx_t for idx_t in sorted_idxs[1:] if distances[idx_t] <= threshold_dist]
# B) Check whether people are looking inwards and whether there are no intrusions
# Deterministic
if n_samples < 2:
for idx_t in indices:
@ -47,8 +50,6 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
dds = torch.tensor(dds).view(-1, 1)
stds = torch.tensor(stds).view(-1, 1)
# stds_te = get_task_error(dds) # similar results to MonoLoco but lower true positive
# print(f'ML : {float(torch.mean(stds))}\n')
# print(f'Task Error: {float(torch.mean(stds_te))}')
laplace_d = torch.cat((dds, stds), dim=1)
samples_d = laplace_sampling(laplace_d, n_samples=n_samples)
@ -93,19 +94,20 @@ def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False
mu_1 = np.array([centers[idx_t][0] + radius * math.cos(theta1), centers[idx_t][1] - radius * math.sin(theta1)])
o_c = (mu_0 + mu_1) / 2
# Verify they are looking inwards.
# 1) Verify they are looking inwards.
# The distance between mus and the center should be less wrt the original position and the center
d_new = np.linalg.norm(mu_0 - mu_1) / 2 if social_distance else np.linalg.norm(mu_0 - mu_1)
d_0 = np.linalg.norm(x_0 - o_c)
d_1 = np.linalg.norm(x_1 - o_c)
# Verify no intrusion for third parties
# 2) Verify no intrusion for third parties
if other_centers.size:
other_distances = np.linalg.norm(other_centers - o_c.reshape(1, -1), axis=1)
else:
other_distances = 100 * np.ones((1, 1)) # Condition verified if no other people
# Binary Classification
# if np.min(other_distances) > radius: # Ablation without orientation
if d_new <= min(d_0, d_1) and np.min(other_distances) > radius:
return True
return False

63
monstereo/eval/eval_kitti.py
View File

@ -25,34 +25,45 @@ class EvalKitti:
'27', '29', '31', '49')
ALP_THRESHOLDS = ('<0.5m', '<1m', '<2m')
OUR_METHODS = ['geometric', 'monoloco', 'monoloco_pp', 'pose', 'reid', 'monstereo']
METHODS_MONO = ['m3d', 'monopsr', 'monodis', 'smoke']
METHODS_MONO = ['m3d', 'monopsr', 'smoke', 'monodis']
METHODS_STEREO = ['3dop', 'psf', 'pseudo-lidar', 'e2e', 'oc-stereo']
BASELINES = ['task_error', 'pixel_error']
HEADERS = ('method', '<0.5', '<1m', '<2m', 'easy', 'moderate', 'hard', 'all')
CATEGORIES = ('pedestrian',)
methods = OUR_METHODS + METHODS_MONO + METHODS_STEREO
def __init__(self, thresh_iou_monoloco=0.3, thresh_iou_base=0.3, thresh_conf_monoloco=0.2, thresh_conf_base=0.5,
verbose=False):
# Set directories
main_dir = os.path.join('data', 'kitti')
dir_gt = os.path.join(main_dir, 'gt')
path_train = os.path.join('splits', 'kitti_train.txt')
path_val = os.path.join('splits', 'kitti_val.txt')
dir_logs = os.path.join('data', 'logs')
assert os.path.exists(dir_logs), "No directory to save final statistics"
dir_fig = os.path.join('data', 'figures')
assert os.path.exists(dir_logs), "No directory to save figures"
self.main_dir = os.path.join('data', 'kitti')
self.dir_gt = os.path.join(self.main_dir, 'gt')
self.methods = self.OUR_METHODS + self.METHODS_MONO + self.METHODS_STEREO
path_train = os.path.join('splits', 'kitti_train.txt')
path_val = os.path.join('splits', 'kitti_val.txt')
dir_logs = os.path.join('data', 'logs')
assert dir_logs, "No directory to save final statistics"
# Set thresholds to obtain comparable recalls
thresh_iou_monoloco = 0.3
thresh_iou_base = 0.3
thresh_conf_monoloco = 0.2
thresh_conf_base = 0.5
def __init__(self, args):
self.verbose = args.verbose
self.net = args.net
self.save = args.save
self.show = args.show
now = datetime.datetime.now()
now_time = now.strftime("%Y%m%d-%H%M")[2:]
self.path_results = os.path.join(dir_logs, 'eval-' + now_time + '.json')
self.verbose = verbose
self.path_results = os.path.join(self.dir_logs, 'eval-' + now_time + '.json')
self.dic_thresh_iou = {method: (thresh_iou_monoloco if method in self.OUR_METHODS
else thresh_iou_base)
for method in self.methods}
self.dic_thresh_conf = {method: (thresh_conf_monoloco if method in self.OUR_METHODS
else thresh_conf_base)
for method in self.methods}
# Set thresholds for comparable recalls
self.dic_thresh_iou = {method: (self.thresh_iou_monoloco if method in self.OUR_METHODS else self.thresh_iou_base)
for method in self.methods}
self.dic_thresh_conf = {method: (self.thresh_conf_monoloco if method in self.OUR_METHODS else self.thresh_conf_base)
for method in self.methods}
# Set thresholds to obtain comparable recall
self.dic_thresh_conf['monopsr'] += 0.4
@ -63,7 +74,7 @@ class EvalKitti:
# Extract validation images for evaluation
names_gt = tuple(os.listdir(self.dir_gt))
_, self.set_val = split_training(names_gt, path_train, path_val)
_, self.set_val = split_training(names_gt, self.path_train, self.path_val)
# self.set_val = ('002282.txt', )
@ -130,12 +141,14 @@ class EvalKitti:
print('\n' + self.category.upper() + ':')
self.show_statistics()
def printer(self, show, save):
if save or show:
show_results(self.dic_stats, self.CLUSTERS, show=show, save=save)
show_spread(self.dic_stats, self.CLUSTERS, show=show, save=save)
show_box_plot(self.errors, self.CLUSTERS, show=show, save=save)
show_task_error(show=show, save=save)
def printer(self):
if self.save or self.show:
show_results(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
show_spread(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
if self.net == 'monstero':
show_box_plot(self.errors, self.CLUSTERS, self.dir_fig, show=self.show, save=self.save)
else:
show_task_error(self.dir_fig, show=self.show, save=self.save)
def _parse_txts(self, path, method):

6
monstereo/eval/generate_kitti.py
View File

@ -240,10 +240,10 @@ def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco', ca
if mode == 'monstereo':
conf_scale = 0.03
elif mode == 'monoloco_pp':
# conf_scale = 0.033
conf_scale = 0.035 # nuScenes for having same recall
conf_scale = 0.033
# conf_scale = 0.035 # nuScenes for having same recall
else:
conf_scale = 0.055
conf_scale = 0.05
conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2))
output_list = [alpha] + uv_box[:-1] + hwl + cam_0 + [ry, conf, bi, epi]

4
monstereo/run.py
View File

@ -179,9 +179,9 @@ def main():
if args.dataset == 'kitti':
from .eval import EvalKitti
kitti_eval = EvalKitti(verbose=args.verbose)
kitti_eval = EvalKitti(args)
kitti_eval.run()
kitti_eval.printer(show=args.show, save=args.save)
kitti_eval.printer()
elif 'nuscenes' in args.dataset:
from .train import Trainer

104
monstereo/visuals/figures.py
View File

@ -17,21 +17,22 @@ DPI = 200
GRID_WIDTH = 0.5
def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=False, stereo=True):
def show_results(dic_stats, clusters, net, dir_fig, show=False, save=False):
"""
Visualize error as function of the distance and compare it with target errors based on human height analyses
"""
phase = 'test'
x_min = 3
x_max = 42
# x_max = 42
x_max = 31
y_min = 0
# y_max = 2.2
y_max = 3.5 if stereo else 5.2
y_max = 3.5 if net == 'monstereo' else 2.6
xx = np.linspace(x_min, x_max, 100)
excl_clusters = ['all', 'easy', 'moderate', 'hard']
excl_clusters = ['all', 'easy', 'moderate', 'hard', '49']
clusters = [clst for clst in clusters if clst not in excl_clusters]
styles = printing_styles(stereo)
styles = printing_styles(net)
for idx_style, style in enumerate(styles.items()):
plt.figure(idx_style, figsize=FIGSIZE)
plt.grid(linewidth=GRID_WIDTH)
@ -51,7 +52,7 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F
if method in ('monstereo', 'pseudo-lidar'):
for i, x in enumerate(xxs):
plt.text(x, errs[i], str(cnts[i]), fontsize=FONTSIZE)
if not stereo:
if net == 'monoloco_pp':
plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5)
# if stereo:
# yy_stereo = get_pixel_error(xx)
@ -62,18 +63,18 @@ def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=F
plt.yticks(fontsize=FONTSIZE)
if save:
plt.tight_layout()
mode = 'stereo' if stereo else 'mono'
path_fig = os.path.join(dir_out, 'results_' + mode + '.png')
path_fig = os.path.join(dir_fig, 'results_' + net + '.png')
plt.savefig(path_fig, dpi=DPI)
print("Figure of results " + mode + " saved in {}".format(path_fig))
print("Figure of results " + net + " saved in {}".format(path_fig))
if show:
plt.show()
plt.close('all')
def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=False):
def show_spread(dic_stats, clusters, net, dir_fig, show=False, save=False):
"""Predicted confidence intervals and task error as a function of ground-truth distance"""
assert net in ('monoloco_pp', 'monstereo'), "network not recognized"
phase = 'test'
excl_clusters = ['all', 'easy', 'moderate', 'hard']
clusters = [clst for clst in clusters if clst not in excl_clusters]
@ -81,42 +82,42 @@ def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=Fa
x_max = 42
y_min = 0
for method in ('monoloco_pp', 'monstereo'):
plt.figure(2, figsize=FIGSIZE)
xxs = get_distances(clusters)
bbs = np.array([dic_stats[phase][method][key]['std_ale'] for key in clusters[:-1]])
if method == 'monoloco_pp':
y_max = 5
color = 'deepskyblue'
epis = np.array([dic_stats[phase][method][key]['std_epi'] for key in clusters[:-1]])
plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)")
else:
y_max = 3.5
color = 'b'
plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
xx = np.linspace(x_min, x_max, 100)
plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
plt.figure(2, figsize=FIGSIZE)
xxs = get_distances(clusters)
bbs = np.array([dic_stats[phase][net][key]['std_ale'] for key in clusters[:-1]])
xx = np.linspace(x_min, x_max, 100)
if net == 'monoloco_pp':
y_max = 5
color = 'deepskyblue'
epis = np.array([dic_stats[phase][net][key]['std_epi'] for key in clusters[:-1]])
plt.plot(xxs, epis, marker='o', color='coral', label="Combined uncertainty (\u03C3)")
else:
y_max = 3.5
color = 'b'
plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.grid(linewidth=GRID_WIDTH)
plt.legend(prop={'size': FONTSIZE})
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save:
plt.tight_layout()
path_fig = os.path.join(dir_out, 'spread_' + method + '.png')
plt.savefig(path_fig, dpi=DPI)
print("Figure of confidence intervals saved in {}".format(path_fig))
if show:
plt.show()
plt.close('all')
plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.grid(linewidth=GRID_WIDTH)
plt.legend(prop={'size': FONTSIZE})
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save:
plt.tight_layout()
path_fig = os.path.join(dir_fig, 'spread_' + net + '.png')
plt.savefig(path_fig, dpi=DPI)
print("Figure of confidence intervals saved in {}".format(path_fig))
if show:
plt.show()
plt.close('all')
def show_task_error(show, save, dir_out='data/figures'):
def show_task_error(dir_fig, show, save):
"""Task error figure"""
plt.figure(3, figsize=FIGSIZE)
xx = np.linspace(0.1, 50, 100)
@ -147,7 +148,7 @@ def show_task_error(show, save, dir_out='data/figures'):
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save:
path_fig = os.path.join(dir_out, 'task_error.png')
path_fig = os.path.join(dir_fig, 'task_error.png')
plt.savefig(path_fig, dpi=DPI)
print("Figure of task error saved in {}".format(path_fig))
if show:
@ -181,7 +182,7 @@ def show_method(save, dir_out='data/figures'):
plt.close('all')
def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save=False):
def show_box_plot(dic_errors, clusters, dir_fig, show=False, save=False):
import pandas as pd
excl_clusters = ['all', 'easy', 'moderate', 'hard']
clusters = [int(clst) for clst in clusters if clst not in excl_clusters]
@ -205,7 +206,7 @@ def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save
plt.ylim(y_min, y_max)
if save:
path_fig = os.path.join(dir_out, 'box_plot_' + name + '.png')
path_fig = os.path.join(dir_fig, 'box_plot_' + name + '.png')
plt.tight_layout()
plt.savefig(path_fig, dpi=DPI)
print("Figure of box plot saved in {}".format(path_fig))
@ -300,8 +301,8 @@ def get_percentile(dist_gmm):
# mad_d = np.mean(np.abs(dist_d - mu_d))
def printing_styles(stereo):
if stereo:
def printing_styles(net):
if net == 'monstereo':
style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'],
"methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'],
"mks": ['s', 'p', 'o', 'v', '*', '^'],
@ -309,11 +310,12 @@ def printing_styles(stereo):
"colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'],
"lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']}
else:
style = {"labels": ['Mono3D', 'Geometric Baseline', 'MonoPSR', '3DOP (stereo)', 'MonoLoco', 'Monoloco++'],
"methods": ['m3d', 'geometric', 'monopsr', '3dop', 'monoloco', 'monoloco_pp'],
style = {"labels": ['Geometric Baseline', 'MonoPSR', 'MonoDIS', '3DOP (stereo)',
'MonoLoco', 'Monoloco++'],
"methods": ['geometric', 'monopsr', 'monodis', '3dop', 'monoloco', 'monoloco_pp'],
"mks": ['*', '^', 'p', '.', 's', 'o', 'o'],
"mksizes": [6, 6, 6, 6, 6, 6], "lws": [1.5, 1.5, 1.5, 1.5, 1.5, 2.2],
"colors": ['r', 'purple', 'olive', 'darkorange', 'b', 'darkblue'],
"colors": ['purple', 'olive', 'r', 'darkorange', 'b', 'darkblue'],
"lstyles": ['solid', 'solid', 'solid', 'dashdot', 'solid', 'solid', ]}
return style