From 10874caea09873793649ea866f879f4a030832c3 Mon Sep 17 00:00:00 2001 From: lorenzo Date: Thu, 25 Jul 2019 18:31:31 +0200 Subject: [PATCH] update task error --- monoloco/eval/eval_kitti.py | 15 ++++++++------- monoloco/utils/__init__.py | 2 +- monoloco/utils/misc.py | 2 +- monoloco/visuals/paper.py | 15 +++++++++------ monoloco/visuals/results.py | 22 ++++++++++------------ 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/monoloco/eval/eval_kitti.py b/monoloco/eval/eval_kitti.py index bc887cb..746480e 100644 --- a/monoloco/eval/eval_kitti.py +++ b/monoloco/eval/eval_kitti.py @@ -25,7 +25,7 @@ class EvalKitti: CLUSTERS = ('easy', 'moderate', 'hard', 'all', '6', '10', '15', '20', '25', '30', '40', '50', '>50') METHODS = ['m3d', 'geom', 'task_error', '3dop', 'our'] HEADERS = ['method', '<0.5', '<1m', '<2m', 'easy', 'moderate', 'hard', 'all'] - CATEGORIES = ['pedestrian', 'cyclist'] + CATEGORIES = ['pedestrian'] def __init__(self, thresh_iou_our=0.3, thresh_iou_m3d=0.3, thresh_conf_m3d=0.3, thresh_conf_our=0.3, verbose=False, stereo=False): @@ -100,15 +100,14 @@ class EvalKitti: if out_gt[0]: out_m3d = self._parse_txts(path_m3d, category, method='m3d') out_3dop = self._parse_txts(path_3dop, category, method='3dop') - # out_md = self._parse_txts(path_md, category, method='md') - out_md = out_m3d + out_md = self._parse_txts(path_md, category, method='md') out_our = self._parse_txts(path_our, category, method='our') out_our_stereo = self._parse_txts(path_our_stereo, category, method='our') if self.stereo else [] # Compute the error with ground truth self._estimate_error(out_gt, out_m3d, method='m3d') self._estimate_error(out_gt, out_3dop, method='3dop') - # self._estimate_error(out_gt, out_md, method='md') + self._estimate_error(out_gt, out_md, method='md') self._estimate_error(out_gt, out_our, method='our') if self.stereo: self._estimate_error(out_gt, out_our_stereo, method='our_stereo') @@ -126,6 +125,9 @@ class EvalKitti: print('\n' + category.upper() + ':') self.show_statistics() + # Show/save results + self.printer(show=False) + def printer(self, show): print_results(self.dic_stats, show) @@ -214,7 +216,7 @@ class EvalKitti: if method == 'our': self.update_errors(dds_geom[idx], dds_gt[idx_gt], cat, self.errors['geom']) self.update_uncertainty(stds_ale[idx], stds_epi[idx], dds[idx], dds_gt[idx_gt], cat) - dd_task_error = dds_gt[idx_gt] + (get_task_error(dds_gt[idx_gt], mode='mad'))**2 + dd_task_error = dds_gt[idx_gt] + (get_task_error(dds_gt[idx_gt]))**2 self.update_errors(dd_task_error, dds_gt[idx_gt], cat, self.errors['task_error']) elif method == 'our_stereo': @@ -248,8 +250,7 @@ class EvalKitti: self.update_errors(dds_our[idx], dd_gt, cat, self.errors['our_merged']) self.update_errors(dds_geom[idx], dd_gt, cat, self.errors['geom_merged']) - self.update_errors(dd_gt + get_task_error(dd_gt, mode='mad'), - dd_gt, cat, self.errors['task_error_merged']) + self.update_errors(dd_gt + get_task_error(dd_gt), dd_gt, cat, self.errors['task_error_merged']) self.update_errors(dds_m3d[indices[0]], dd_gt, cat, self.errors['m3d_merged']) self.update_errors(dds_3dop[indices[1]], dd_gt, cat, self.errors['3dop_merged']) self.update_errors(dds_md[indices[2]], dd_gt, cat, self.errors['md_merged']) diff --git a/monoloco/utils/__init__.py b/monoloco/utils/__init__.py index e894156..3e82b01 100644 --- a/monoloco/utils/__init__.py +++ b/monoloco/utils/__init__.py @@ -5,4 +5,4 @@ from .kitti import check_conditions, get_category, split_training, parse_ground_ from .camera import xyz_from_distance, get_keypoints, pixel_to_camera, project_3d from .logs import set_logger from .stereo import depth_from_disparity -from ..utils.nuscenes import select_categories +from ..utils.nuscenes import select_categories \ No newline at end of file diff --git a/monoloco/utils/misc.py b/monoloco/utils/misc.py index 465d377..b1c59f7 100644 --- a/monoloco/utils/misc.py +++ b/monoloco/utils/misc.py @@ -26,7 +26,7 @@ def append_cluster(dic_jo, phase, xx, dd, kps): def get_task_error(dd): """Get target error not knowing the gender, modeled through a Gaussian Mixure model""" - mm = 0.0459 + mm = 0.046 return dd * mm diff --git a/monoloco/visuals/paper.py b/monoloco/visuals/paper.py index 3a4e6f3..fa9a42e 100644 --- a/monoloco/visuals/paper.py +++ b/monoloco/visuals/paper.py @@ -62,8 +62,8 @@ def paper(): plt.plot(xx, yy_female, '-.', linewidth=1.7, color='darkorange', label='Adult female') plt.plot(xx, yy_male, '-.', linewidth=1.7, color='b', label='Adult male') plt.xlim(np.min(xx), np.max(xx)) - plt.xlabel("Distance from the camera [m]") - plt.ylabel("Localization error due to human height variation [m]") + plt.xlabel("Ground-truth distance from the camera $d_{gt}$ [m]") + plt.ylabel("Localization error $\hat{e}$ due to human height variation [m]") plt.legend(loc=(0.01, 0.55)) # Location from 0 to 1 from lower left plt.savefig(os.path.join('docs', 'task_error.png')) @@ -74,6 +74,7 @@ def target_error(xx, mm): def gmm(): dist_gmm, dist_male, dist_female = height_distributions() + # get_percentile(dist_gmm) mu_gmm = np.mean(dist_gmm) mm_gmm = np.mean(np.abs(1 - mu_gmm / dist_gmm)) mm_male = np.mean(np.abs(1 - np.mean(dist_male) / dist_male)) @@ -136,10 +137,12 @@ def plot_dist(dist_gmm, dist_men, dist_women): def get_percentile(dist_gmm): - dd_gt = 100 + dd_gt = 1000 mu_gmm = np.mean(dist_gmm) dist_d = dd_gt * mu_gmm / dist_gmm - perc_d, _ = np.nanpercentile(dist_d, [18.5, 81.5]) # Laplace bi => 63% + perc_d, _ = np.nanpercentile(dist_d, [18.5, 81.5]) # Laplace bi => 63% + perc_d2, _ = np.nanpercentile(dist_d, [23, 77]) mu_d = np.mean(dist_d) - mm_bi = (mu_d - perc_d) / mu_d - mad_d = np.mean(np.abs(dist_d - mu_d)) + # mm_bi = (mu_d - perc_d) / mu_d + # mm_test = (mu_d - perc_d2) / mu_d + # mad_d = np.mean(np.abs(dist_d - mu_d)) diff --git a/monoloco/visuals/results.py b/monoloco/visuals/results.py index e205f48..5bc42ab 100644 --- a/monoloco/visuals/results.py +++ b/monoloco/visuals/results.py @@ -6,6 +6,8 @@ import numpy as np import matplotlib.pyplot as plt from matplotlib.patches import Ellipse +from ..utils import get_task_error + def print_results(dic_stats, show=False): @@ -22,14 +24,13 @@ def print_results(dic_stats, show=False): x_min = 0 x_max = 38 xx = np.linspace(0, 60, 100) - mm_gender = 0.0556 excl_clusters = ['all', '50', '>50', 'easy', 'moderate', 'hard'] clusters = tuple([clst for clst in dic_stats[phase]['our'] if clst not in excl_clusters]) - yy_gender = target_error(xx, mm_gender) + yy_gender = get_task_error(xx) yy_gps = np.linspace(5., 5., xx.shape[0]) plt.figure(0) - fig_name = 'results.png' + plt.grid(linewidth=0.2) plt.xlabel("Distance [meters]") plt.ylabel("Average localization error [m]") plt.xlim(x_min, x_max) @@ -52,7 +53,7 @@ def print_results(dic_stats, show=False): if show: plt.show() else: - plt.savefig(os.path.join(dir_out, fig_name)) + plt.savefig(os.path.join(dir_out, 'results.png')) plt.close() # SPREAD b Figure @@ -67,7 +68,7 @@ def print_results(dic_stats, show=False): bbs = np.array([dic_stats[phase]['our'][key]['std_ale'] for key in clusters]) xxs = get_distances(clusters) - yys = target_error(np.array(xxs), mm_gender) + yys = get_task_error(np.array(xxs)) ax[1].plot(xxs, bbs, marker='s', color='b', label="Spread b") ax[1].plot(xxs, yys, '--', color='lightgreen', label="Task error", linewidth=2.5) yys_up = [rec_c + ar/2 * scale * yy for yy in yys] @@ -76,8 +77,8 @@ def print_results(dic_stats, show=False): bbs_down = [rec_c - ar/2 * scale * bb for bb in bbs] if plots_line: - ax[0].plot(xxs, yys_up, '--', color='lightgreen', markersize=5, linewidth=1) - ax[0].plot(xxs, yys_down, '--', color='lightgreen', markersize=5, linewidth=1) + ax[0].plot(xxs, yys_up, '--', color='lightgreen', markersize=5, linewidth=1.4) + ax[0].plot(xxs, yys_down, '--', color='lightgreen', markersize=5, linewidth=1.4) ax[0].plot(xxs, bbs_up, marker='s', color='b', markersize=5, linewidth=0.7) ax[0].plot(xxs, bbs_down, marker='s', color='b', markersize=5, linewidth=0.7) @@ -94,14 +95,11 @@ def print_results(dic_stats, show=False): plt.legend() if show: plt.show() + else: + plt.savefig(os.path.join(dir_out, 'spread_bi.png')) plt.close() -def target_error(xx, mm): - """Multiplication""" - return mm * xx - - def get_distances(clusters): """Extract distances as intermediate values between 2 clusters"""