charles/monoloco
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Visualization (#2)

Browse Source 
* update readme

* remove legend for front

* change default z_max to 100m

* add visualization for distance

* add visualization for distance

* adjust text

* remove legend for frontal image

* change frontal visualization

* adjust visualization

* update fontsize as constant

* change name of saving

* change architecture name

* change architecture name

* change architecture name

* remove gt boxes

* support different resolutions

* adapt to front and combined

* change name to multi
This commit is contained in:
Lorenzo Bertoni 2020-11-30 11:49:47 +01:00 committed by GitHub
parent f8d968a831
commit 4992d4c34e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 291 additions and 506 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -5,7 +5,7 @@ This repository contains the code for three research projects:
1. **MonStereo: When Monocular and Stereo Meet at the Tail of 3D Human Localization** 1. **MonStereo: When Monocular and Stereo Meet at the Tail of 3D Human Localization**
[README](https://github.com/vita-epfl/monstereo/tree/master/docs/MonStereo.md) & [Article](https://arxiv.org/abs/2008.10913) [README](https://github.com/vita-epfl/monstereo/tree/master/docs/MonStereo.md) & [Article](https://arxiv.org/abs/2008.10913)
![moonstereo](docs/out_005523.png) ![monstereo](docs/out_005523.png)
2. **Perceiving Humans: from Monocular 3D Localization to Social Distancing** 2. **Perceiving Humans: from Monocular 3D Localization to Social Distancing**
[README](https://github.com/vita-epfl/monstereo/tree/master/docs/SocialDistancing.md) & [Article](https://arxiv.org/abs/2009.00984) [README](https://github.com/vita-epfl/monstereo/tree/master/docs/SocialDistancing.md) & [Article](https://arxiv.org/abs/2009.00984)

9
docs/MonStereo.md
View File

@ -104,7 +104,7 @@ and runs MonStereo for 3d location of the detected poses.
Output options include json files and/or visualization of the predictions on the image in *frontal mode*, Output options include json files and/or visualization of the predictions on the image in *frontal mode*,
*birds-eye-view mode* or *combined mode* and can be specified with `--output_types` *birds-eye-view mode* or *multi mode* and can be specified with `--output_types`
### Ground truth matching ### Ground truth matching
@ -121,13 +121,13 @@ show all the prediction for the image
After downloading model and ground-truth file, a demo can be tested with the following commands: After downloading model and ground-truth file, a demo can be tested with the following commands:
`python3 -m monstereo.run predict --glob docs/000840*.png --output_types combined --scale 2 `python3 -m monstereo.run predict --glob docs/000840*.png --output_types multi --scale 2
--model data/models/ms-200710-1511.pkl --z_max 30 --checkpoint resnet152 --path_gt data/arrays/names-kitti-200615-1022.json --model data/models/ms-200710-1511.pkl --z_max 30 --checkpoint resnet152 --path_gt data/arrays/names-kitti-200615-1022.json
-o data/output` -o data/output`
![Crowded scene](out_000840.png) ![Crowded scene](out_000840.png)
`python3 -m monstereo.run predict --glob docs/005523*.png --output_types combined --scale 2 `python3 -m monstereo.run predict --glob docs/005523*.png --output_types multi --scale 2
--model data/models/ms-200710-1511.pkl --z_max 30 --checkpoint resnet152 --path_gt data/arrays/names-kitti-200615-1022.json --model data/models/ms-200710-1511.pkl --z_max 30 --checkpoint resnet152 --path_gt data/arrays/names-kitti-200615-1022.json
-o data/output` -o data/output`
@ -138,8 +138,7 @@ Preprocessing and training step are already fully supported by the code provided
but require first to run a pose detector over but require first to run a pose detector over
all the training images and collect the annotations. all the training images and collect the annotations.
The code supports this option (by running the predict script and using `--mode pifpaf`). The code supports this option (by running the predict script and using `--mode pifpaf`).
Once the code will be made publicly available, we will add
links to download annotations.
### Datasets ### Datasets
Download KITTI ground truth files and camera calibration matrices for training Download KITTI ground truth files and camera calibration matrices for training

8
monstereo/eval/eval_kitti.py
View File

@ -126,10 +126,10 @@ class EvalKitti:
def printer(self, show, save): def printer(self, show, save):
if save or show: if save or show:
show_results(self.dic_stats, self.CLUSTERS, show, save) show_results(self.dic_stats, self.CLUSTERS, show=show, save=save)
show_spread(self.dic_stats, self.CLUSTERS, show, save) show_spread(self.dic_stats, self.CLUSTERS, show=show, save=save)
show_box_plot(self.errors, self.CLUSTERS, show, save) show_box_plot(self.errors, self.CLUSTERS, show=show, save=save)
show_task_error(show, save) show_task_error(show=show, save=save)
def _parse_txts(self, path, method): def _parse_txts(self, path, method):

262
monstereo/network/architectures.py
View File

@ -3,10 +3,10 @@ import torch
import torch.nn as nn import torch.nn as nn
class SimpleModel(nn.Module): class MonStereoModel(nn.Module):
def __init__(self, input_size, output_size=2, linear_size=512, p_dropout=0.2, num_stage=3, device='cuda'): def __init__(self, input_size, output_size=2, linear_size=512, p_dropout=0.2, num_stage=3, device='cuda'):
super(SimpleModel, self).__init__() super(MonStereoModel, self).__init__()
self.num_stage = num_stage self.num_stage = num_stage
self.stereo_size = input_size self.stereo_size = input_size
@ -102,264 +102,6 @@ class MyLinearSimple(nn.Module):
return out return out
class DecisionModel(nn.Module):
def __init__(self, input_size, output_size=2, linear_size=512, p_dropout=0.2, num_stage=3, device='cuda:1'):
super(DecisionModel, self).__init__()
self.num_stage = num_stage
self.stereo_size = input_size
self.mono_size = int(input_size / 2)
self.output_size = output_size - 1
self.linear_size = linear_size
self.p_dropout = p_dropout
self.num_stage = num_stage
self.linear_stages_mono, self.linear_stages_stereo, self.linear_stages_dec = [], [], []
self.device = device
# Initialize weights
# ------------------------Stereo----------------------------------------------
# Preprocessing
self.w1_stereo = nn.Linear(self.stereo_size, self.linear_size)
self.batch_norm_stereo = nn.BatchNorm1d(self.linear_size)
# Internal loop
for _ in range(num_stage):
self.linear_stages_stereo.append(MyLinear_stereo(self.linear_size, self.p_dropout))
self.linear_stages_stereo = nn.ModuleList(self.linear_stages_stereo)
# Post processing
self.w2_stereo = nn.Linear(self.linear_size, self.output_size)
# ------------------------Mono----------------------------------------------
# Preprocessing
self.w1_mono = nn.Linear(self.mono_size, self.linear_size)
self.batch_norm_mono = nn.BatchNorm1d(self.linear_size)
# Internal loop
for _ in range(num_stage):
self.linear_stages_mono.append(MyLinear_stereo(self.linear_size, self.p_dropout))
self.linear_stages_mono = nn.ModuleList(self.linear_stages_mono)
# Post processing
self.w2_mono = nn.Linear(self.linear_size, self.output_size)
# ------------------------Decision----------------------------------------------
# Preprocessing
self.w1_dec = nn.Linear(self.stereo_size, self.linear_size)
self.batch_norm_dec = nn.BatchNorm1d(self.linear_size)
#
# Internal loop
for _ in range(num_stage):
self.linear_stages_dec.append(MyLinear(self.linear_size, self.p_dropout))
self.linear_stages_dec = nn.ModuleList(self.linear_stages_dec)
# Post processing
self.w2_dec = nn.Linear(self.linear_size, 1)
# ------------------------Other----------------------------------------------
# NO-weight operations
self.relu = nn.ReLU(inplace=True)
self.dropout = nn.Dropout(self.p_dropout)
def forward(self, x, label=None):
# Mono
y_m = self.w1_mono(x[:, 0:34])
y_m = self.batch_norm_mono(y_m)
y_m = self.relu(y_m)
y_m = self.dropout(y_m)
for i in range(self.num_stage):
y_m = self.linear_stages_mono[i](y_m)
y_m = self.w2_mono(y_m)
# Stereo
y_s = self.w1_stereo(x)
y_s = self.batch_norm_stereo(y_s)
y_s = self.relu(y_s)
y_s = self.dropout(y_s)
for i in range(self.num_stage):
y_s = self.linear_stages_stereo[i](y_s)
y_s = self.w2_stereo(y_s)
# Decision
y_d = self.w1_dec(x)
y_d = self.batch_norm_dec(y_d)
y_d = self.relu(y_d)
y_d = self.dropout(y_d)
for i in range(self.num_stage):
y_d = self.linear_stages_dec[i](y_d)
aux = self.w2_dec(y_d)
# Combine
if label is not None:
gate = label
else:
gate = torch.where(torch.sigmoid(aux) > 0.3,
torch.tensor([1.]).to(self.device), torch.tensor([0.]).to(self.device))
y = gate * y_s + (1-gate) * y_m
# Cat with auxiliary task
y = torch.cat((y, aux), dim=1)
return y
class AttentionModel(nn.Module):
def __init__(self, input_size, output_size=2, linear_size=512, p_dropout=0.2, num_stage=3, device='cuda'):
super(AttentionModel, self).__init__()
self.num_stage = num_stage
self.stereo_size = input_size
self.mono_size = int(input_size / 2)
self.output_size = output_size - 1
self.linear_size = linear_size
self.p_dropout = p_dropout
self.num_stage = num_stage
self.linear_stages_mono, self.linear_stages_stereo, self.linear_stages_comb = [], [], []
self.device = device
# Initialize weights
# ------------------------Stereo----------------------------------------------
# Preprocessing
self.w1_stereo = nn.Linear(self.stereo_size, self.linear_size)
self.batch_norm_stereo = nn.BatchNorm1d(self.linear_size)
# Internal loop
for _ in range(num_stage):
self.linear_stages_stereo.append(MyLinear_stereo(self.linear_size, self.p_dropout))
self.linear_stages_stereo = nn.ModuleList(self.linear_stages_stereo)
# Post processing
self.w2_stereo = nn.Linear(self.linear_size, self.linear_size)
# ------------------------Mono----------------------------------------------
# Preprocessing
self.w1_mono = nn.Linear(self.mono_size, self.linear_size)
self.batch_norm_mono = nn.BatchNorm1d(self.linear_size)
# Internal loop
for _ in range(num_stage):
self.linear_stages_mono.append(MyLinear_stereo(self.linear_size, self.p_dropout))
self.linear_stages_mono = nn.ModuleList(self.linear_stages_mono)
# Post processing
self.w2_mono = nn.Linear(self.linear_size, self.linear_size)
# ------------------------Combined----------------------------------------------
# Preprocessing
self.w1_comb = nn.Linear(self.linear_size, self.linear_size)
self.batch_norm_comb = nn.BatchNorm1d(self.linear_size)
#
# Internal loop
for _ in range(num_stage):
self.linear_stages_comb.append(MyLinear(self.linear_size, self.p_dropout))
self.linear_stages_comb = nn.ModuleList(self.linear_stages_comb)
# Post processing
self.w2_comb = nn.Linear(self.linear_size, self.linear_size)
# ------------------------Other----------------------------------------------
# Auxiliary
self.w_aux = nn.Linear(self.linear_size, 1)
# Final
self.w_fin = nn.Linear(self.linear_size, self.output_size)
# NO-weight operations
self.relu = nn.ReLU(inplace=True)
self.dropout = nn.Dropout(self.p_dropout)
def forward(self, x, label=None):
# Mono
y_m = self.w1_mono(x[:, 0:34])
y_m = self.batch_norm_mono(y_m)
y_m = self.relu(y_m)
y_m = self.dropout(y_m)
for i in range(self.num_stage):
y_m = self.linear_stages_mono[i](y_m)
y_m = self.w2_mono(y_m)
# Stereo
y_s = self.w1_stereo(x)
y_s = self.batch_norm_stereo(y_s)
y_s = self.relu(y_s)
y_s = self.dropout(y_s)
for i in range(self.num_stage):
y_s = self.linear_stages_stereo[i](y_s)
y_s = self.w2_stereo(y_s)
# Auxiliary task
aux = self.w_aux(y_s)
# Combined
if label is not None:
gate = label
else:
gate = torch.where(torch.sigmoid(aux) > 0.3,
torch.tensor([1.]).to(self.device), torch.tensor([0.]).to(self.device))
y_c = gate * y_s + (1-gate) * y_m
y_c = self.w1_comb(y_c)
y_c = self.batch_norm_comb(y_c)
y_c = self.relu(y_c)
y_c = self.dropout(y_c)
y_c = self.w_fin(y_c)
# Cat with auxiliary task
y = torch.cat((y_c, aux), dim=1)
return y
class MyLinear_stereo(nn.Module):
def __init__(self, linear_size, p_dropout=0.5):
super(MyLinear_stereo, self).__init__()
self.l_size = linear_size
self.relu = nn.ReLU(inplace=True)
self.dropout = nn.Dropout(p_dropout)
# self.w0_a = nn.Linear(self.l_size, self.l_size)
# self.batch_norm0_a = nn.BatchNorm1d(self.l_size)
# self.w0_b = nn.Linear(self.l_size, self.l_size)
# self.batch_norm0_b = nn.BatchNorm1d(self.l_size)
self.w1 = nn.Linear(self.l_size, self.l_size)
self.batch_norm1 = nn.BatchNorm1d(self.l_size)
self.w2 = nn.Linear(self.l_size, self.l_size)
self.batch_norm2 = nn.BatchNorm1d(self.l_size)
def forward(self, x):
#
# x = self.w0_a(x)
# x = self.batch_norm0_a(x)
# x = self.w0_b(x)
# x = self.batch_norm0_b(x)
y = self.w1(x)
y = self.batch_norm1(y)
y = self.relu(y)
y = self.dropout(y)
y = self.w2(y)
y = self.batch_norm2(y)
y = self.relu(y)
y = self.dropout(y)
out = x + y
return out
class MonolocoModel(nn.Module): class MonolocoModel(nn.Module):
""" """
Architecture inspired by https://github.com/una-dinosauria/3d-pose-baseline Architecture inspired by https://github.com/una-dinosauria/3d-pose-baseline

4
monstereo/network/net.py
View File

@ -14,7 +14,7 @@ import torch
from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance
from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\ from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\
filter_outputs, cluster_outputs, unnormalize_bi filter_outputs, cluster_outputs, unnormalize_bi
from .architectures import MonolocoModel, SimpleModel from .architectures import MonolocoModel, MonStereoModel
class Loco: class Loco:
@ -55,7 +55,7 @@ class Loco:
self.model = MonolocoModel(p_dropout=p_dropout, input_size=input_size, linear_size=linear_size, self.model = MonolocoModel(p_dropout=p_dropout, input_size=input_size, linear_size=linear_size,
output_size=output_size) output_size=output_size)
else: else:
self.model = SimpleModel(p_dropout=p_dropout, input_size=input_size, output_size=output_size, self.model = MonStereoModel(p_dropout=p_dropout, input_size=input_size, output_size=output_size,
linear_size=linear_size, device=self.device) linear_size=linear_size, device=self.device)
self.model.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage)) self.model.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage))

18
monstereo/predict.py
View File

@ -53,10 +53,11 @@ def predict(args):
image_paths, images, processed_images_cpu, fields_batch)): image_paths, images, processed_images_cpu, fields_batch)):
if args.output_directory is None: if args.output_directory is None:
output_path = image_paths[0] splits = os.path.split(image_paths[0])
output_path = os.path.join(splits[0], 'out_' + splits[1])
else: else:
file_name = os.path.basename(image_paths[0]) file_name = os.path.basename(image_paths[0])
output_path = os.path.join(args.output_directory, file_name) output_path = os.path.join(args.output_directory, 'out_' + file_name)
print('image', idx, image_path, output_path) print('image', idx, image_path, output_path)
keypoint_sets, scores, pifpaf_out = pifpaf.forward(image, processed_image_cpu, fields) keypoint_sets, scores, pifpaf_out = pifpaf.forward(image, processed_image_cpu, fields)
@ -133,17 +134,12 @@ def factory_outputs(args, images_outputs, output_path, pifpaf_outputs, dic_out=N
skeleton_painter.keypoints(ax, keypoint_sets, scores=scores) skeleton_painter.keypoints(ax, keypoint_sets, scores=scores)
else: else:
if any((xx in args.output_types for xx in ['front', 'bird', 'combined'])): if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
epistemic = False print(output_path)
if args.n_dropout > 0:
epistemic = True
if dic_out['boxes']: # Only print in case of detections if dic_out['boxes']: # Only print in case of detections
printer = Printer(images_outputs[1], output_path, kk, output_types=args.output_types printer = Printer(images_outputs[1], output_path, kk, args)
, z_max=args.z_max, epistemic=epistemic)
figures, axes = printer.factory_axes() figures, axes = printer.factory_axes()
printer.draw(figures, axes, dic_out, images_outputs[1], show_all=args.show_all, draw_box=args.draw_box, printer.draw(figures, axes, dic_out, images_outputs[1])
save=True, show=args.show)
if 'json' in args.output_types: if 'json' in args.output_types:
with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff: with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:

8
monstereo/run.py
View File

@ -35,8 +35,10 @@ def cli():
predict_parser.add_argument('-o', '--output-directory', help='Output directory') predict_parser.add_argument('-o', '--output-directory', help='Output directory')
predict_parser.add_argument('--output_types', nargs='+', default=['json'], predict_parser.add_argument('--output_types', nargs='+', default=['json'],
help='what to output: json keypoints skeleton for Pifpaf' help='what to output: json keypoints skeleton for Pifpaf'
'json bird front combined for Monoloco') 'json bird front or multi for MonStereo')
predict_parser.add_argument('--no_save', help='to show images', action='store_true')
predict_parser.add_argument('--show', help='to show images', action='store_true') predict_parser.add_argument('--show', help='to show images', action='store_true')
predict_parser.add_argument('--dpi', help='image resolution', type=int, default=100)
# Pifpaf # Pifpaf
nets.cli(predict_parser) nets.cli(predict_parser)
@ -49,8 +51,7 @@ def cli():
predict_parser.add_argument('--path_gt', help='path of json file with gt 3d localization', predict_parser.add_argument('--path_gt', help='path of json file with gt 3d localization',
default='data/arrays/names-kitti-200615-1022.json') default='data/arrays/names-kitti-200615-1022.json')
predict_parser.add_argument('--transform', help='transformation for the pose', default='None') predict_parser.add_argument('--transform', help='transformation for the pose', default='None')
predict_parser.add_argument('--draw_box', help='to draw box in the images', action='store_true') predict_parser.add_argument('--z_max', type=int, help='maximum meters distance for predictions', default=100)
predict_parser.add_argument('--z_max', type=int, help='maximum meters distance for predictions', default=22)
predict_parser.add_argument('--n_dropout', type=int, help='Epistemic uncertainty evaluation', default=0) predict_parser.add_argument('--n_dropout', type=int, help='Epistemic uncertainty evaluation', default=0)
predict_parser.add_argument('--dropout', type=float, help='dropout parameter', default=0.2) predict_parser.add_argument('--dropout', type=float, help='dropout parameter', default=0.2)
predict_parser.add_argument('--show_all', help='only predict ground-truth matches or all', action='store_true') predict_parser.add_argument('--show_all', help='only predict ground-truth matches or all', action='store_true')
@ -116,7 +117,6 @@ def main():
predict(args) predict(args)
elif args.command == 'prep': elif args.command == 'prep':
if 'nuscenes' in args.dataset: if 'nuscenes' in args.dataset:
from .prep.preprocess_nu import PreprocessNuscenes from .prep.preprocess_nu import PreprocessNuscenes
prep = PreprocessNuscenes(args.dir_ann, args.dir_nuscenes, args.dataset, args.iou_min) prep = PreprocessNuscenes(args.dir_ann, args.dir_nuscenes, args.dataset, args.iou_min)

8
monstereo/train/trainer.py
View File

@ -23,7 +23,7 @@ from torch.optim import lr_scheduler
from .datasets import KeypointsDataset from .datasets import KeypointsDataset
from .losses import CompositeLoss, MultiTaskLoss, AutoTuneMultiTaskLoss from .losses import CompositeLoss, MultiTaskLoss, AutoTuneMultiTaskLoss
from ..network import extract_outputs, extract_labels from ..network import extract_outputs, extract_labels
from ..network.architectures import SimpleModel from ..network.architectures import MonStereoModel
from ..utils import set_logger from ..utils import set_logger
@ -97,7 +97,7 @@ class Trainer:
now = datetime.datetime.now() now = datetime.datetime.now()
now_time = now.strftime("%Y%m%d-%H%M")[2:] now_time = now.strftime("%Y%m%d-%H%M")[2:]
name_out = 'ms-' + now_time name_out = 'monstereo-' + now_time
if self.save: if self.save:
self.path_model = os.path.join(dir_out, name_out + '.pkl') self.path_model = os.path.join(dir_out, name_out + '.pkl')
self.logger = set_logger(os.path.join(dir_logs, name_out)) self.logger = set_logger(os.path.join(dir_logs, name_out))
@ -122,8 +122,8 @@ class Trainer:
self.logger.info('Sizes of the dataset: {}'.format(self.dataset_sizes)) self.logger.info('Sizes of the dataset: {}'.format(self.dataset_sizes))
print(">>> creating model") print(">>> creating model")
self.model = SimpleModel(input_size=input_size, output_size=output_size, linear_size=hidden_size, self.model = MonStereoModel(input_size=input_size, output_size=output_size, linear_size=hidden_size,
p_dropout=dropout, num_stage=self.n_stage, device=self.device) p_dropout=dropout, num_stage=self.n_stage, device=self.device)
self.model.to(self.device) self.model.to(self.device)
print(">>> model params: {:.3f}M".format(sum(p.numel() for p in self.model.parameters()) / 1000000.0)) print(">>> model params: {:.3f}M".format(sum(p.numel() for p in self.model.parameters()) / 1000000.0))
print(">>> loss params: {}".format(sum(p.numel() for p in self.mt_loss.parameters()))) print(">>> loss params: {}".format(sum(p.numel() for p in self.mt_loss.parameters())))

92
monstereo/visuals/figures.py
View File

@ -11,31 +11,34 @@ from matplotlib.patches import Ellipse
from ..utils import get_task_error, get_pixel_error from ..utils import get_task_error, get_pixel_error
def show_results(dic_stats, clusters, show=False, save=False, stereo=True): FONTSIZE = 15
FIGSIZE = (9.6, 7.2)
DPI = 200
GRID_WIDTH = 0.5
def show_results(dic_stats, clusters, dir_out='data/figures', show=False, save=False, stereo=True):
""" """
Visualize error as function of the distance and compare it with target errors based on human height analyses Visualize error as function of the distance and compare it with target errors based on human height analyses
""" """
dir_out = 'docs'
phase = 'test' phase = 'test'
x_min = 3 x_min = 3
x_max = 42 x_max = 42
y_min = 0 y_min = 0
# y_max = 2.2 # y_max = 2.2
y_max = 3.5 if stereo else 5.2 y_max = 3.5 if stereo else 5.2
xx = np.linspace(x_min, x_max, 100) xx = np.linspace(x_min, x_max, 100)
excl_clusters = ['all', 'easy', 'moderate', 'hard'] excl_clusters = ['all', 'easy', 'moderate', 'hard']
clusters = [clst for clst in clusters if clst not in excl_clusters] clusters = [clst for clst in clusters if clst not in excl_clusters]
plt.figure(0)
styles = printing_styles(stereo) styles = printing_styles(stereo)
for idx_style, style in enumerate(styles.items()): for idx_style, style in enumerate(styles.items()):
plt.figure(idx_style) plt.figure(idx_style, figsize=FIGSIZE)
plt.grid(linewidth=0.2) plt.grid(linewidth=GRID_WIDTH)
plt.xlim(x_min, x_max) plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max) plt.ylim(y_min, y_max)
plt.xlabel("Ground-truth distance [m]") plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
plt.ylabel("Average localization error (ALE) [m]") plt.ylabel("Average localization error (ALE) [m]", fontsize=FONTSIZE)
for idx, method in enumerate(styles['methods']): for idx, method in enumerate(styles['methods']):
errs = [dic_stats[phase][method][clst]['mean'] for clst in clusters[:-1]] # last cluster only a bound errs = [dic_stats[phase][method][clst]['mean'] for clst in clusters[:-1]] # last cluster only a bound
cnts = [dic_stats[phase][method][clst]['cnt'] for clst in clusters[:-1]] # last cluster only a bound cnts = [dic_stats[phase][method][clst]['cnt'] for clst in clusters[:-1]] # last cluster only a bound
@ -47,30 +50,31 @@ def show_results(dic_stats, clusters, show=False, save=False, stereo=True):
label=styles['labels'][idx], linestyle=styles['lstyles'][idx], color=styles['colors'][idx]) label=styles['labels'][idx], linestyle=styles['lstyles'][idx], color=styles['colors'][idx])
if method in ('monstereo', 'pseudo-lidar'): if method in ('monstereo', 'pseudo-lidar'):
for i, x in enumerate(xxs): for i, x in enumerate(xxs):
plt.text(x, errs[i], str(cnts[i]), fontsize=10) plt.text(x, errs[i], str(cnts[i]), fontsize=FONTSIZE)
if not stereo: if not stereo:
plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5) plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5)
# if stereo: # if stereo:
# yy_stereo = get_pixel_error(xx) # yy_stereo = get_pixel_error(xx)
# plt.plot(xx, yy_stereo, linewidth=1.4, color='k', label='Pixel error') # plt.plot(xx, yy_stereo, linewidth=1.4, color='k', label='Pixel error')
plt.legend(loc='upper left') plt.legend(loc='upper left', prop={'size': FONTSIZE})
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save: if save:
plt.tight_layout() plt.tight_layout()
mode = 'stereo' if stereo else 'mono' mode = 'stereo' if stereo else 'mono'
path_fig = os.path.join(dir_out, 'results_' + mode + '.png') path_fig = os.path.join(dir_out, 'results_' + mode + '.png')
plt.savefig(path_fig) plt.savefig(path_fig, dpi=DPI)
print("Figure of results " + mode + " saved in {}".format(path_fig)) print("Figure of results " + mode + " saved in {}".format(path_fig))
if show: if show:
plt.show() plt.show()
plt.close('all') plt.close('all')
def show_spread(dic_stats, clusters, show=False, save=False): def show_spread(dic_stats, clusters, dir_out='data/figures', show=False, save=False):
"""Predicted confidence intervals and task error as a function of ground-truth distance""" """Predicted confidence intervals and task error as a function of ground-truth distance"""
phase = 'test' phase = 'test'
dir_out = 'docs'
excl_clusters = ['all', 'easy', 'moderate', 'hard'] excl_clusters = ['all', 'easy', 'moderate', 'hard']
clusters = [clst for clst in clusters if clst not in excl_clusters] clusters = [clst for clst in clusters if clst not in excl_clusters]
x_min = 3 x_min = 3
@ -78,7 +82,7 @@ def show_spread(dic_stats, clusters, show=False, save=False):
y_min = 0 y_min = 0
for method in ('monoloco_pp', 'monstereo'): for method in ('monoloco_pp', 'monstereo'):
plt.figure(2) plt.figure(2, figsize=FIGSIZE)
xxs = get_distances(clusters) xxs = get_distances(clusters)
bbs = np.array([dic_stats[phase][method][key]['std_ale'] for key in clusters[:-1]]) bbs = np.array([dic_stats[phase][method][key]['std_ale'] for key in clusters[:-1]])
if method == 'monoloco_pp': if method == 'monoloco_pp':
@ -89,31 +93,32 @@ def show_spread(dic_stats, clusters, show=False, save=False):
else: else:
y_max = 3.5 y_max = 3.5
color = 'b' color = 'b'
plt.plot(xx, get_pixel_error(xx), linewidth=1.4, color='k', label='Pixel error') plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)") plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
xx = np.linspace(x_min, x_max, 100) xx = np.linspace(x_min, x_max, 100)
plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=2.5) plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
plt.xlabel("Ground-truth distance [m]") plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
plt.ylabel("Uncertainty [m]") plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
plt.xlim(x_min, x_max) plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max) plt.ylim(y_min, y_max)
plt.grid(linewidth=0.2) plt.grid(linewidth=GRID_WIDTH)
plt.legend() plt.legend(prop={'size': FONTSIZE})
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save: if save:
plt.tight_layout() plt.tight_layout()
path_fig = os.path.join(dir_out, 'spread_' + method + '.png') path_fig = os.path.join(dir_out, 'spread_' + method + '.png')
plt.savefig(path_fig) plt.savefig(path_fig, dpi=DPI)
print("Figure of confidence intervals saved in {}".format(path_fig)) print("Figure of confidence intervals saved in {}".format(path_fig))
if show: if show:
plt.show() plt.show()
plt.close('all') plt.close('all')
def show_task_error(show, save): def show_task_error(show, save, dir_out='data/figures'):
"""Task error figure""" """Task error figure"""
plt.figure(3) plt.figure(3, figsize=FIGSIZE)
dir_out = 'docs'
xx = np.linspace(0.1, 50, 100) xx = np.linspace(0.1, 50, 100)
mu_men = 178 mu_men = 178
mu_women = 165 mu_women = 165
@ -128,7 +133,7 @@ def show_task_error(show, save):
yy_young_female = target_error(xx, mm_young_female) yy_young_female = target_error(xx, mm_young_female)
yy_gender = target_error(xx, mm_gmm) yy_gender = target_error(xx, mm_gmm)
yy_stereo = get_pixel_error(xx) yy_stereo = get_pixel_error(xx)
plt.grid(linewidth=0.3) plt.grid(linewidth=GRID_WIDTH)
plt.plot(xx, yy_young_male, linestyle='dotted', linewidth=2.1, color='b', label='Adult/young male') plt.plot(xx, yy_young_male, linestyle='dotted', linewidth=2.1, color='b', label='Adult/young male')
plt.plot(xx, yy_young_female, linestyle='dotted', linewidth=2.1, color='darkorange', label='Adult/young female') plt.plot(xx, yy_young_female, linestyle='dotted', linewidth=2.1, color='darkorange', label='Adult/young female')
plt.plot(xx, yy_gender, '--', color='lightgreen', linewidth=2.8, label='Generic adult (task error)') plt.plot(xx, yy_gender, '--', color='lightgreen', linewidth=2.8, label='Generic adult (task error)')
@ -139,20 +144,21 @@ def show_task_error(show, save):
plt.xlabel("Ground-truth distance from the camera $d_{gt}$ [m]") plt.xlabel("Ground-truth distance from the camera $d_{gt}$ [m]")
plt.ylabel("Localization error $\hat{e}$ due to human height variation [m]") # pylint: disable=W1401 plt.ylabel("Localization error $\hat{e}$ due to human height variation [m]") # pylint: disable=W1401
plt.legend(loc=(0.01, 0.55)) # Location from 0 to 1 from lower left plt.legend(loc=(0.01, 0.55)) # Location from 0 to 1 from lower left
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
if save: if save:
path_fig = os.path.join(dir_out, 'task_error.png') path_fig = os.path.join(dir_out, 'task_error.png')
plt.savefig(path_fig) plt.savefig(path_fig, dpi=DPI)
print("Figure of task error saved in {}".format(path_fig)) print("Figure of task error saved in {}".format(path_fig))
if show: if show:
plt.show() plt.show()
plt.close('all') plt.close('all')
def show_method(save): def show_method(save, dir_out='data/figures'):
""" method figure""" """ method figure"""
dir_out = 'docs'
std_1 = 0.75 std_1 = 0.75
fig = plt.figure(1) fig = plt.figure(4, figsize=FIGSIZE)
ax = fig.add_subplot(1, 1, 1) ax = fig.add_subplot(1, 1, 1)
ell_3 = Ellipse((0, 2), width=std_1 * 2, height=0.3, angle=-90, color='b', fill=False, linewidth=2.5) ell_3 = Ellipse((0, 2), width=std_1 * 2, height=0.3, angle=-90, color='b', fill=False, linewidth=2.5)
ell_4 = Ellipse((0, 2), width=std_1 * 3, height=0.3, angle=-90, color='r', fill=False, ell_4 = Ellipse((0, 2), width=std_1 * 3, height=0.3, angle=-90, color='r', fill=False,
@ -164,42 +170,44 @@ def show_method(save):
plt.plot([0, -3], [0, 4], 'k--') plt.plot([0, -3], [0, 4], 'k--')
plt.xlim(-3, 3) plt.xlim(-3, 3)
plt.ylim(0, 3.5) plt.ylim(0, 3.5)
plt.xticks([]) plt.xticks(fontsize=FONTSIZE)
plt.yticks([]) plt.yticks(fontsize=FONTSIZE)
plt.xlabel('X [m]') plt.xlabel('X [m]')
plt.ylabel('Z [m]') plt.ylabel('Z [m]')
if save: if save:
path_fig = os.path.join(dir_out, 'output_method.png') path_fig = os.path.join(dir_out, 'output_method.png')
plt.savefig(path_fig) plt.savefig(path_fig, dpi=DPI)
print("Figure of method saved in {}".format(path_fig)) print("Figure of method saved in {}".format(path_fig))
plt.close('all')
def show_box_plot(dic_errors, clusters, show=False, save=False): def show_box_plot(dic_errors, clusters, dir_out='data/figures', show=False, save=False):
import pandas as pd import pandas as pd
dir_out = 'docs'
excl_clusters = ['all', 'easy', 'moderate', 'hard'] excl_clusters = ['all', 'easy', 'moderate', 'hard']
clusters = [int(clst) for clst in clusters if clst not in excl_clusters] clusters = [int(clst) for clst in clusters if clst not in excl_clusters]
methods = ('monstereo', 'pseudo-lidar', '3dop', 'monoloco') methods = ('monstereo', 'pseudo-lidar', '3dop', 'monoloco')
y_min = 0 y_min = 0
y_max = 25 # 18 for the other y_max = 16 # 18 for the other
xxs = get_distances(clusters) xxs = get_distances(clusters)
labels = [str(xx) for xx in xxs] labels = [str(xx) for xx in xxs]
for idx, method in enumerate(methods): for idx, method in enumerate(methods):
df = pd.DataFrame([dic_errors[method][str(clst)] for clst in clusters[:-1]]).T df = pd.DataFrame([dic_errors[method][str(clst)] for clst in clusters[:-1]]).T
df.columns = labels df.columns = labels
plt.figure(idx) plt.figure(idx, figsize=FIGSIZE) # with 200 dpi it becomes 1920x1440
_ = df.boxplot() _ = df.boxplot()
name = 'MonStereo' if method == 'monstereo' else method name = 'MonStereo' if method == 'monstereo' else method
plt.title(name) plt.title(name, fontsize=FONTSIZE)
plt.ylabel('Average localization error (ALE) [m]') plt.ylabel('Average localization error (ALE) [m]', fontsize=FONTSIZE)
plt.xlabel('Ground-truth distance [m]') plt.xlabel('Ground-truth distance [m]', fontsize=FONTSIZE)
plt.xticks(fontsize=FONTSIZE)
plt.yticks(fontsize=FONTSIZE)
plt.ylim(y_min, y_max) plt.ylim(y_min, y_max)
if save: if save:
path_fig = os.path.join(dir_out, 'box_plot_' + name + '.png') path_fig = os.path.join(dir_out, 'box_plot_' + name + '.png')
plt.tight_layout() plt.tight_layout()
plt.savefig(path_fig) plt.savefig(path_fig, dpi=DPI)
print("Figure of box plot saved in {}".format(path_fig)) print("Figure of box plot saved in {}".format(path_fig))
if show: if show:
plt.show() plt.show()
@ -297,7 +305,7 @@ def printing_styles(stereo):
style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'], style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'],
"methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'], "methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'],
"mks": ['s', 'p', 'o', 'v', '*', '^'], "mks": ['s', 'p', 'o', 'v', '*', '^'],
"mksizes": [6, 6, 6, 6, 6, 6], "lws": [1.2, 1.2, 1.2, 1.2, 1.3, 1.5], "mksizes": [6, 6, 6, 6, 6, 6], "lws": [2, 2, 2, 2, 2, 2.2],
"colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'], "colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'],
"lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']} "lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']}
else: else:

384
monstereo/visuals/printer.py
View File

@ -1,50 +1,70 @@
""" """
Class for drawing frontal, bird-eye-view and combined figures Class for drawing frontal, bird-eye-view and multi figures
""" """
# pylint: disable=attribute-defined-outside-init # pylint: disable=attribute-defined-outside-init
import math import math
from collections import OrderedDict from collections import OrderedDict
import numpy as np
import matplotlib
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.cm as cm from matplotlib.patches import Rectangle
from matplotlib.patches import Ellipse, Circle, Rectangle
from mpl_toolkits.axes_grid1 import make_axes_locatable
from ..utils import pixel_to_camera, get_task_error from ..utils import pixel_to_camera
def get_angle(xx, zz):
"""Obtain the points to plot the confidence of each annotation"""
theta = math.atan2(zz, xx)
angle = theta * (180 / math.pi)
return angle
def image_attributes(dpi, output_types):
c = 0.7 if 'front' in output_types else 1.0
return dict(dpi=dpi,
fontsize_d=round(14 * c),
fontsize_bv=round(24 * c),
fontsize_num=round(22 * c),
fontsize_ax=round(16 * c),
linewidth=round(8 * c),
markersize=round(16 * c),
y_box_margin=round(24 * math.sqrt(c)),
stereo=dict(color='deepskyblue',
numcolor='darkorange',
linewidth=1 * c),
mono=dict(color='red',
numcolor='firebrick',
linewidth=2 * c)
)
class Printer: class Printer:
""" """
Print results on images: birds eye view and computed distance Print results on images: birds eye view and computed distance
""" """
FONTSIZE_BV = 16 FIG_WIDTH = 15
FONTSIZE = 18 extensions = []
TEXTCOLOR = 'darkorange' y_scale = 1
COLOR_KPS = 'yellow' nones = lambda n: [None for _ in range(n)]
mpl_im0, stds_ale, stds_epi, xx_gt, zz_gt, xx_pred, zz_pred, dd_real, uv_centers, uv_shoulders, uv_kps, boxes, \
boxes_gt, uv_camera, radius, auxs = nones(16)
def __init__(self, image, output_path, kk, output_types, epistemic=False, z_max=30, fig_width=10): def __init__(self, image, output_path, kk, args):
self.im = image self.im = image
self.kk = kk
self.output_types = output_types
self.epistemic = epistemic
self.z_max = z_max # To include ellipses in the image
self.y_scale = 1
self.width = self.im.size[0] self.width = self.im.size[0]
self.height = self.im.size[1] self.height = self.im.size[1]
self.fig_width = fig_width
# Define the output dir
self.output_path = output_path self.output_path = output_path
self.cmap = cm.get_cmap('jet') self.kk = kk
self.extensions = [] self.output_types = args.output_types
self.z_max = args.z_max # To include ellipses in the image
self.show_all = args.show_all
self.show = args.show_all
self.save = not args.no_save
# Define variables of the class to change for every image # define image attributes
self.mpl_im0 = self.stds_ale = self.stds_epi = self.xx_gt = self.zz_gt = self.xx_pred = self.zz_pred =\ self.attr = image_attributes(args.dpi, args.output_types)
self.dds_real = self.uv_centers = self.uv_shoulders = self.uv_kps = self.boxes = self.boxes_gt = \
self.uv_camera = self.radius = self.auxs = None
def _process_results(self, dic_ann): def _process_results(self, dic_ann):
# Include the vectors inside the interval given by z_max # Include the vectors inside the interval given by z_max
@ -59,39 +79,38 @@ class Printer:
for idx, xx in enumerate(dic_ann['xyz_real'])] for idx, xx in enumerate(dic_ann['xyz_real'])]
self.zz_pred = [xx[2] if xx[2] < self.z_max - self.stds_epi[idx] else 0 self.zz_pred = [xx[2] if xx[2] < self.z_max - self.stds_epi[idx] else 0
for idx, xx in enumerate(dic_ann['xyz_pred'])] for idx, xx in enumerate(dic_ann['xyz_pred'])]
self.dd_pred = dic_ann['dds_pred']
self.dds_real = dic_ann['dds_real'] self.dd_real = dic_ann['dds_real']
self.uv_heads = dic_ann['uv_heads']
self.uv_shoulders = dic_ann['uv_shoulders'] self.uv_shoulders = dic_ann['uv_shoulders']
self.boxes = dic_ann['boxes'] self.boxes = dic_ann['boxes']
self.boxes_gt = dic_ann['boxes_gt'] self.boxes_gt = dic_ann['boxes_gt']
self.uv_camera = (int(self.im.size[0] / 2), self.im.size[1]) self.uv_camera = (int(self.im.size[0] / 2), self.im.size[1])
self.radius = 11 / 1600 * self.width
if dic_ann['aux']: if dic_ann['aux']:
self.auxs = dic_ann['aux'] if dic_ann['aux'] else None self.auxs = dic_ann['aux'] if dic_ann['aux'] else None
def factory_axes(self): def factory_axes(self):
"""Create axes for figures: front bird combined""" """Create axes for figures: front bird multi"""
axes = [] axes = []
figures = [] figures = []
# Initialize combined figure, resizing it for aesthetic proportions # Initialize multi figure, resizing it for aesthetic proportion
if 'combined' in self.output_types: if 'multi' in self.output_types:
assert 'bird' and 'front' not in self.output_types, \ assert 'bird' and 'front' not in self.output_types, \
"combined figure cannot be print together with front or bird ones" "multi figure cannot be print together with front or bird ones"
self.y_scale = self.width / (self.height * 2) # Defined proportion self.y_scale = self.width / (self.height * 2) # Defined proportion
if self.y_scale < 0.95 or self.y_scale > 1.05: # allows more variation without resizing if self.y_scale < 0.95 or self.y_scale > 1.05: # allows more variation without resizing
self.im = self.im.resize((self.width, round(self.height * self.y_scale))) self.im = self.im.resize((self.width, round(self.height * self.y_scale)))
self.width = self.im.size[0] self.width = self.im.size[0]
self.height = self.im.size[1] self.height = self.im.size[1]
fig_width = self.fig_width + 0.6 * self.fig_width fig_width = self.FIG_WIDTH + 0.6 * self.FIG_WIDTH
fig_height = self.fig_width * self.height / self.width fig_height = self.FIG_WIDTH * self.height / self.width
# Distinguish between KITTI images and general images # Distinguish between KITTI images and general images
fig_ar_1 = 0.8 fig_ar_1 = 0.8
width_ratio = 1.9 width_ratio = 1.9
self.extensions.append('.combined.png') self.extensions.append('.multi.png')
fig, (ax0, ax1) = plt.subplots(1, 2, sharey=False, gridspec_kw={'width_ratios': [width_ratio, 1]}, fig, (ax0, ax1) = plt.subplots(1, 2, sharey=False, gridspec_kw={'width_ratios': [width_ratio, 1]},
figsize=(fig_width, fig_height)) figsize=(fig_width, fig_height))
@ -101,12 +120,12 @@ class Printer:
figures.append(fig) figures.append(fig)
assert 'front' not in self.output_types and 'bird' not in self.output_types, \ assert 'front' not in self.output_types and 'bird' not in self.output_types, \
"--combined arguments is not supported with other visualizations" "--multi arguments is not supported with other visualizations"
# Initialize front figure # Initialize front figure
elif 'front' in self.output_types: elif 'front' in self.output_types:
width = self.fig_width width = self.FIG_WIDTH
height = self.fig_width * self.height / self.width height = self.FIG_WIDTH * self.height / self.width
self.extensions.append(".front.png") self.extensions.append(".front.png")
plt.figure(0) plt.figure(0)
fig0, ax0 = plt.subplots(1, 1, figsize=(width, height)) fig0, ax0 = plt.subplots(1, 1, figsize=(width, height))
@ -114,18 +133,8 @@ class Printer:
figures.append(fig0) figures.append(fig0)
# Create front figure axis # Create front figure axis
if any(xx in self.output_types for xx in ['front', 'combined']): if any(xx in self.output_types for xx in ['front', 'multi']):
ax0 = self.set_axes(ax0, axis=0) ax0 = self._set_axes(ax0, axis=0)
divider = make_axes_locatable(ax0)
cax = divider.append_axes('right', size='3%', pad=0.05)
bar_ticks = self.z_max // 5 + 1
norm = matplotlib.colors.Normalize(vmin=0, vmax=self.z_max)
scalar_mappable = plt.cm.ScalarMappable(cmap=self.cmap, norm=norm)
scalar_mappable.set_array([])
plt.colorbar(scalar_mappable, ticks=np.linspace(0, self.z_max, bar_ticks),
boundaries=np.arange(- 0.05, self.z_max + 0.1, .1), cax=cax, label='Z [m]')
axes.append(ax0) axes.append(ax0)
if not axes: if not axes:
axes.append(None) axes.append(None)
@ -136,64 +145,114 @@ class Printer:
fig1, ax1 = plt.subplots(1, 1) fig1, ax1 = plt.subplots(1, 1)
fig1.set_tight_layout(True) fig1.set_tight_layout(True)
figures.append(fig1) figures.append(fig1)
if any(xx in self.output_types for xx in ['bird', 'combined']): if any(xx in self.output_types for xx in ['bird', 'multi']):
ax1 = self.set_axes(ax1, axis=1) # Adding field of view ax1 = self._set_axes(ax1, axis=1) # Adding field of view
axes.append(ax1) axes.append(ax1)
return figures, axes return figures, axes
def draw(self, figures, axes, dic_out, image, show_all=False, draw_text=True, legend=True, draw_box=False, def draw(self, figures, axes, dic_out, image):
save=False, show=False):
# Process the annotation dictionary of monoloco # Process the annotation dictionary of monoloco
self._process_results(dic_out) self._process_results(dic_out)
# whether to include instances that don't match the ground-truth # whether to include instances that don't match the ground-truth
iterator = range(len(self.zz_pred)) if show_all else range(len(self.zz_gt)) iterator = range(len(self.zz_pred)) if self.show_all else range(len(self.zz_gt))
if not iterator: if not iterator:
print("-"*110 + '\n' + "! No instances detected, be sure to include file with ground-truth values or " print("-" * 110 + '\n' + "! No instances detected, be sure to include file with ground-truth values or "
"use the command --show_all" + '\n' + "-"*110) "use the command --show_all" + '\n' + "-" * 110)
# Draw the front figure # Draw the front figure
num = 0 number = dict(flag=False, num=97)
if 'multi' in self.output_types:
number['flag'] = True # add numbers
self.mpl_im0.set_data(image) self.mpl_im0.set_data(image)
for idx in iterator: for idx in iterator:
if any(xx in self.output_types for xx in ['front', 'combined']) and self.zz_pred[idx] > 0: if any(xx in self.output_types for xx in ['front', 'multi']) and self.zz_pred[idx] > 0:
self._draw_front(axes[0],
color = self.cmap((self.zz_pred[idx] % self.z_max) / self.z_max) self.dd_pred[idx],
self.draw_circle(axes, self.uv_shoulders[idx], color) idx,
if draw_box: number)
self.draw_boxes(axes, idx, color) number['num'] += 1
if draw_text:
self.draw_text_front(axes, self.uv_shoulders[idx], num)
num += 1
# Draw the bird figure # Draw the bird figure
num = 0 number['num'] = 97
for idx in iterator: for idx in iterator:
if any(xx in self.output_types for xx in ['bird', 'combined']) and self.zz_pred[idx] > 0: if any(xx in self.output_types for xx in ['bird', 'multi']) and self.zz_pred[idx] > 0:
# Draw ground truth and uncertainty # Draw ground truth and uncertainty
self.draw_uncertainty(axes, idx) self._draw_uncertainty(axes, idx)
# Draw bird eye view text # Draw bird eye view text
if draw_text: if number['flag']:
self.draw_text_bird(axes, idx, num) self._draw_text_bird(axes, idx, number['num'])
num += 1 number['num'] += 1
# Add the legend self._draw_legend(axes)
if legend:
draw_legend(axes)
# Draw, save or/and show the figures # Draw, save or/and show the figures
for idx, fig in enumerate(figures): for idx, fig in enumerate(figures):
fig.canvas.draw() fig.canvas.draw()
if save: if self.save:
fig.savefig(self.output_path + self.extensions[idx], bbox_inches='tight') fig.savefig(self.output_path + self.extensions[idx], bbox_inches='tight', dpi=self.attr['dpi'])
if show: if self.show:
fig.show() fig.show()
plt.close(fig) plt.close(fig)
def draw_uncertainty(self, axes, idx): def _draw_front(self, ax, z, idx, number):
mode = 'stereo' if self.auxs[idx] > 0.3 else 'mono'
# Bbox
w = min(self.width-2, self.boxes[idx][2] - self.boxes[idx][0])
h = min(self.height-2, (self.boxes[idx][3] - self.boxes[idx][1]) * self.y_scale)
x0 = self.boxes[idx][0]
y0 = self.boxes[idx][1] * self.y_scale
y1 = y0 + h
rectangle = Rectangle((x0, y0),
width=w,
height=h,
fill=False,
color=self.attr[mode]['color'],
linewidth=self.attr[mode]['linewidth'])
ax.add_patch(rectangle)
z_str = str(z).split(sep='.')
text = z_str[0] + '.' + z_str[1][0]
bbox_config = {'facecolor': self.attr[mode]['color'], 'alpha': 0.4, 'linewidth': 0}
x_t = x0 - 1.5
y_t = y1 + self.attr['y_box_margin']
if y_t < (self.height-10):
ax.annotate(
text,
(x_t, y_t),
fontsize=self.attr['fontsize_d'],
weight='bold',
xytext=(5.0, 5.0),
textcoords='offset points',
color='white',
bbox=bbox_config,
)
if number['flag']:
ax.text(x0 - 17,
y1 + 14,
chr(number['num']),
fontsize=self.attr['fontsize_num'],
color=self.attr[mode]['numcolor'],
weight='bold')
def _draw_text_bird(self, axes, idx, num):
"""Plot the number in the bird eye view map"""
mode = 'stereo' if self.auxs[idx] > 0.3 else 'mono'
std = self.stds_epi[idx] if self.stds_epi[idx] > 0 else self.stds_ale[idx]
theta = math.atan2(self.zz_pred[idx], self.xx_pred[idx])
delta_x = std * math.cos(theta)
delta_z = std * math.sin(theta)
axes[1].text(self.xx_pred[idx] + delta_x + 0.2, self.zz_pred[idx] + delta_z + 0/2, chr(num),
fontsize=self.attr['fontsize_bv'],
color=self.attr[mode]['numcolor'])
def _draw_uncertainty(self, axes, idx):
theta = math.atan2(self.zz_pred[idx], self.xx_pred[idx]) theta = math.atan2(self.zz_pred[idx], self.xx_pred[idx])
dic_std = {'ale': self.stds_ale[idx], 'epi': self.stds_epi[idx]} dic_std = {'ale': self.stds_ale[idx], 'epi': self.stds_epi[idx]}
@ -208,89 +267,84 @@ class Printer:
# MonoLoco # MonoLoco
if not self.auxs: if not self.auxs:
axes[1].plot(dic_x['epi'], dic_y['epi'], color='coral', linewidth=2, label="Epistemic Uncertainty") axes[1].plot(dic_x['epi'],
axes[1].plot(dic_x['ale'], dic_y['ale'], color='deepskyblue', linewidth=4, label="Aleatoric Uncertainty") dic_y['epi'],
axes[1].plot(self.xx_pred[idx], self.zz_pred[idx], color='cornflowerblue', label="Prediction", markersize=6, color='coral',
linewidth=round(self.attr['linewidth']/2),
label="Epistemic Uncertainty")
axes[1].plot(dic_x['ale'],
dic_y['ale'],
color='deepskyblue',
linewidth=self.attr['linewidth'],
label="Aleatoric Uncertainty")
axes[1].plot(self.xx_pred[idx],
self.zz_pred[idx],
color='cornflowerblue',
label="Prediction",
markersize=self.attr['markersize'],
marker='o') marker='o')
if self.gt[idx]: if self.gt[idx]:
axes[1].plot(self.xx_gt[idx], self.zz_gt[idx], axes[1].plot(self.xx_gt[idx],
color='k', label="Ground-truth", markersize=8, marker='x') self.zz_gt[idx],
color='k',
label="Ground-truth",
markersize=8,
marker='x')
# MonStereo(stereo case) # MonStereo(stereo case)
elif self.auxs[idx] > 0.5: elif self.auxs[idx] > 0.5:
axes[1].plot(dic_x['ale'], dic_y['ale'], color='r', linewidth=4, label="Prediction (mono)") axes[1].plot(dic_x['ale'],
axes[1].plot(dic_x['ale'], dic_y['ale'], color='deepskyblue', linewidth=4, label="Prediction (stereo+mono)") dic_y['ale'],
color='r',
linewidth=self.attr['linewidth'],
label="Prediction (mono)")
axes[1].plot(dic_x['ale'],
dic_y['ale'],
color='deepskyblue',
linewidth=self.attr['linewidth'],
label="Prediction (stereo+mono)")
if self.gt[idx]: if self.gt[idx]:
axes[1].plot(self.xx_gt[idx], self.zz_gt[idx], axes[1].plot(self.xx_gt[idx],
color='k', label="Ground-truth", markersize=8, marker='x') self.zz_gt[idx],
color='k',
label="Ground-truth",
markersize=self.attr['markersize'],
marker='x')
# MonStereo (monocular case) # MonStereo (monocular case)
else: else:
axes[1].plot(dic_x['ale'], dic_y['ale'], color='deepskyblue', linewidth=4, label="Prediction (stereo+mono)") axes[1].plot(dic_x['ale'],
axes[1].plot(dic_x['ale'], dic_y['ale'], color='r', linewidth=4, label="Prediction (mono)") dic_y['ale'],
color='deepskyblue',
linewidth=self.attr['linewidth'],
label="Prediction (stereo+mono)")
axes[1].plot(dic_x['ale'],
dic_y['ale'],
color='r',
linewidth=self.attr['linewidth'],
label="Prediction (mono)")
if self.gt[idx]: if self.gt[idx]:
axes[1].plot(self.xx_gt[idx], self.zz_gt[idx], axes[1].plot(self.xx_gt[idx],
color='k', label="Ground-truth", markersize=8, marker='x') self.zz_gt[idx],
color='k',
label="Ground-truth",
markersize=self.attr['markersize'],
marker='x')
def draw_ellipses(self, axes, idx): def _draw_legend(self, axes):
"""draw uncertainty ellipses""" # Bird eye view legend
target = get_task_error(self.dds_real[idx]) if any(xx in self.output_types for xx in ['bird', 'multi']):
angle_gt = get_angle(self.xx_gt[idx], self.zz_gt[idx]) handles, labels = axes[1].get_legend_handles_labels()
ellipse_real = Ellipse((self.xx_gt[idx], self.zz_gt[idx]), width=target * 2, height=1, by_label = OrderedDict(zip(labels, handles))
angle=angle_gt, color='lightgreen', fill=True, label="Task error") axes[1].legend(by_label.values(), by_label.keys(), loc='best', prop={'size': 15})
axes[1].add_patch(ellipse_real)
if abs(self.zz_gt[idx] - self.zz_pred[idx]) > 0.001:
axes[1].plot(self.xx_gt[idx], self.zz_gt[idx], 'kx', label="Ground truth", markersize=3)
angle = get_angle(self.xx_pred[idx], self.zz_pred[idx]) def _set_axes(self, ax, axis):
ellipse_ale = Ellipse((self.xx_pred[idx], self.zz_pred[idx]), width=self.stds_ale[idx] * 2,
height=1, angle=angle, color='b', fill=False, label="Aleatoric Uncertainty",
linewidth=1.3)
ellipse_var = Ellipse((self.xx_pred[idx], self.zz_pred[idx]), width=self.stds_epi[idx] * 2,
height=1, angle=angle, color='r', fill=False, label="Uncertainty",
linewidth=1, linestyle='--')
axes[1].add_patch(ellipse_ale)
if self.epistemic:
axes[1].add_patch(ellipse_var)
axes[1].plot(self.xx_pred[idx], self.zz_pred[idx], 'ro', label="Predicted", markersize=3)
def draw_boxes(self, axes, idx, color):
ww_box = self.boxes[idx][2] - self.boxes[idx][0]
hh_box = (self.boxes[idx][3] - self.boxes[idx][1]) * self.y_scale
ww_box_gt = self.boxes_gt[idx][2] - self.boxes_gt[idx][0]
hh_box_gt = (self.boxes_gt[idx][3] - self.boxes_gt[idx][1]) * self.y_scale
rectangle = Rectangle((self.boxes[idx][0], self.boxes[idx][1] * self.y_scale),
width=ww_box, height=hh_box, fill=False, color=color, linewidth=3)
rectangle_gt = Rectangle((self.boxes_gt[idx][0], self.boxes_gt[idx][1] * self.y_scale),
width=ww_box_gt, height=hh_box_gt, fill=False, color='g', linewidth=2)
axes[0].add_patch(rectangle_gt)
axes[0].add_patch(rectangle)
def draw_text_front(self, axes, uv, num):
axes[0].text(uv[0] + self.radius, uv[1] * self.y_scale - self.radius, str(num),
fontsize=self.FONTSIZE, color=self.TEXTCOLOR, weight='bold')
def draw_text_bird(self, axes, idx, num):
"""Plot the number in the bird eye view map"""
std = self.stds_epi[idx] if self.stds_epi[idx] > 0 else self.stds_ale[idx]
theta = math.atan2(self.zz_pred[idx], self.xx_pred[idx])
delta_x = std * math.cos(theta)
delta_z = std * math.sin(theta)
axes[1].text(self.xx_pred[idx] + delta_x, self.zz_pred[idx] + delta_z,
str(num), fontsize=self.FONTSIZE_BV, color='darkorange')
def draw_circle(self, axes, uv, color):
circle = Circle((uv[0], uv[1] * self.y_scale), radius=self.radius, color=color, fill=True)
axes[0].add_patch(circle)
def set_axes(self, ax, axis):
assert axis in (0, 1) assert axis in (0, 1)
if axis == 0: if axis == 0:
@ -308,23 +362,9 @@ class Printer:
corr = round(float(x_max / 3)) corr = round(float(x_max / 3))
ax.plot([0, x_max], [0, self.z_max], 'k--') ax.plot([0, x_max], [0, self.z_max], 'k--')
ax.plot([0, -x_max], [0, self.z_max], 'k--') ax.plot([0, -x_max], [0, self.z_max], 'k--')
ax.set_xlim(-x_max+corr, x_max-corr) ax.set_xlim(-x_max + corr, x_max - corr)
ax.set_ylim(0, self.z_max+1) ax.set_ylim(0, self.z_max + 1)
ax.set_xlabel("X [m]") ax.set_xlabel("X [m]")
plt.xticks(fontsize=self.attr['fontsize_ax'])
plt.yticks(fontsize=self.attr['fontsize_ax'])
return ax return ax
def draw_legend(axes):
handles, labels = axes[1].get_legend_handles_labels()
by_label = OrderedDict(zip(labels, handles))
axes[1].legend(by_label.values(), by_label.keys(), loc='best')
def get_angle(xx, zz):
"""Obtain the points to plot the confidence of each annotation"""
theta = math.atan2(zz, xx)
angle = theta * (180 / math.pi)
return angle

2
tests/test_package.py
View File

@ -44,7 +44,7 @@ def tst_printer(dic_out, kk, image_path):
"""Draw a fake figure""" """Draw a fake figure"""
with open(image_path, 'rb') as f: with open(image_path, 'rb') as f:
pil_image = Image.open(f).convert('RGB') pil_image = Image.open(f).convert('RGB')
printer = Printer(image=pil_image, output_path='tests/test_image', kk=kk, output_types=['combined'], z_max=15) printer = Printer(image=pil_image, output_path='tests/test_image', kk=kk, output_types=['multi'], z_max=15)
figures, axes = printer.factory_axes() figures, axes = printer.factory_axes()
printer.draw(figures, axes, dic_out, pil_image, save=True) printer.draw(figures, axes, dic_out, pil_image, save=True)