charles/monoloco
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Camera parameters (#5)

Browse Source 
* change reorder flag

* make social distance as separate function

* temp

* temp

* refactor names pifpaf outputs

* verify conflicting options

* add logging

* custom camera parameters

* convert back to print

* add pyc files
This commit is contained in:
Lorenzo Bertoni 2021-02-09 10:53:15 +01:00 committed by GitHub
parent 8bd4de53ac
commit 23ab2f05aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 79 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,7 +2,7 @@
data/
.DS_store
__pycache__
Monoloco/*.pyc
monstereo/*.pyc
.pytest*
build/
dist/

2
monstereo/__init__.py
View File

@ -1,4 +1,4 @@
"""Open implementation of MonoLoco++ / MonStereo."""
__version__ = '0.2.0'
__version__ = '0.2.1'

9
monstereo/activity.py
View File

@ -111,18 +111,11 @@ def show_social(args, image_t, output_path, annotations, dic_out):
assert 'front' in args.output_types or 'bird' in args.output_types, "outputs allowed: front and/or bird"
angles = dic_out['angles']
dds = dic_out['dds_pred']
stds = dic_out['stds_ale']
xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
# Prepare color for social distancing
colors = ['r' if social_interactions(idx, xz_centers, angles, dds,
stds=stds,
threshold_prob=args.threshold_prob,
threshold_dist=args.threshold_dist,
radii=args.radii)
else 'deepskyblue'
for idx, _ in enumerate(dic_out['xyz_pred'])]
colors = ['r' if flag else 'deepskyblue' for flag in dic_out['social_distance']]
# Draw keypoints and orientation
if 'front' in args.output_types:

20
monstereo/network/net.py
View File

@ -14,6 +14,7 @@ import torch
from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance
from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\
filter_outputs, cluster_outputs, unnormalize_bi
from ..activity import social_interactions
from .architectures import MonolocoModel, MonStereoModel
@ -172,7 +173,7 @@ class Loco:
if verbose:
print("NO ground-truth associated")
if reorder:
if reorder and matches:
matches = reorder_matches(matches, boxes, mode='left_right')
all_idxs = [idx for idx, _ in matches] + not_matches
@ -233,6 +234,23 @@ class Loco:
dic_out['xyz_real'].append(xyz_real.squeeze().tolist())
return dic_out
@staticmethod
def social_distance(dic_out, args):
angles = dic_out['angles']
dds = dic_out['dds_pred']
stds = dic_out['stds_ale']
xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
# Prepare color for social distancing
dic_out['social_distance'] = [bool(social_interactions(idx, xz_centers, angles, dds,
stds=stds,
threshold_prob=args.threshold_prob,
threshold_dist=args.threshold_dist,
radii=args.radii))
for idx, _ in enumerate(dic_out['xyz_pred'])]
return dic_out
def median_disparity(dic_out, keypoints, keypoints_r, mask):
"""

30
monstereo/network/process.py
View File

@ -1,11 +1,15 @@
import json
import os
import logging
import numpy as np
import torch
import torchvision
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
from ..utils import get_keypoints, pixel_to_camera, to_cartesian, back_correct_angles
BF = 0.54 * 721
@ -13,6 +17,9 @@ z_min = 4
z_max = 60
D_MIN = BF / z_max
D_MAX = BF / z_min
FL = 5.7 # nuScenes focal length (mm)
Sx = 7.2 # nuScenes sensor size (mm)
Sy = 5.4 # nuScenes sensor size (mm)
def preprocess_monstereo(keypoints, keypoints_r, kk):
@ -67,31 +74,28 @@ def factory_for_gt(im_size, name=None, path_gt=None, verbose=True):
with open(path_gt, 'r') as f:
dic_names = json.load(f)
if verbose:
print('-' * 120 + "\nGround-truth file opened")
logger.info('-' * 120 + "\nGround-truth file opened")
except (FileNotFoundError, TypeError):
if verbose:
print('-' * 120 + "\nGround-truth file not found")
logger.info('-' * 120 + "\nGround-truth file not found")
dic_names = {}
try:
kk = dic_names[name]['K']
dic_gt = dic_names[name]
if verbose:
print("Matched ground-truth file!")
logger.info("Matched ground-truth file!")
except KeyError:
dic_gt = None
x_factor = im_size[0] / 1600
y_factor = im_size[1] / 900
pixel_factor = (x_factor + y_factor) / 1.75 # 1.75 for MOT
# pixel_factor = 1
if im_size[0] / im_size[1] > 2.5:
if im_size[0] / im_size[1] > 2.5: # KITTI default
kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]] # Kitti calibration
else:
kk = [[1266.4 * pixel_factor, 0., 816.27 * x_factor],
[0, 1266.4 * pixel_factor, 491.5 * y_factor],
[0., 0., 1.]] # nuScenes calibration
else: # nuScenes camera parameters
kk = [
[im_size[0]*FL/Sx, 0., im_size[0]/2],
[0., im_size[1]*FL/Sy, im_size[1]/2],
[0., 0., 1.]]
if verbose:
print("Using a standard calibration matrix...")
logger.info("Using a standard calibration matrix...")
return kk, dic_gt

73
monstereo/predict.py
View File

@ -27,6 +27,7 @@ LOG = logging.getLogger(__name__)
OPENPIFPAF_PATH = 'data/models/shufflenetv2k30-201104-224654-cocokp-d75ed641.pkl' # Default model
def factory_from_args(args):
# Data
@ -40,7 +41,7 @@ def factory_from_args(args):
if os.path.exists(OPENPIFPAF_PATH):
args.checkpoint = OPENPIFPAF_PATH
else:
print("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. "
LOG.info("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. "
"Using a ShuffleNet backbone")
args.checkpoint = 'shufflenetv2k30'
@ -64,7 +65,7 @@ def factory_from_args(args):
# Make default pifpaf argument
args.force_complete_pose = True
print("Force complete pose is active")
LOG.info("Force complete pose is active")
# Configure
decoder.configure(args)
@ -82,7 +83,6 @@ def predict(args):
# Load Models
assert args.net in ('monoloco_pp', 'monstereo', 'pifpaf')
if args.net in ('monoloco_pp', 'monstereo'):
net = Loco(model=args.model, net=args.net, device=args.device, n_dropout=args.n_dropout, p_dropout=args.dropout)
@ -99,15 +99,12 @@ def predict(args):
data, batch_size=args.batch_size, shuffle=False,
pin_memory=False, collate_fn=datasets.collate_images_anns_meta)
# visualizers
annotation_painter = openpifpaf.show.AnnotationPainter()
for batch_i, (image_tensors_batch, _, meta_batch) in enumerate(data_loader):
pred_batch = processor.batch(model, image_tensors_batch, device=args.device)
# unbatch (only for MonStereo)
for idx, (pred, meta) in enumerate(zip(pred_batch, meta_batch)):
LOG.info('batch %d: %s', batch_i, meta['file_name'])
print('batch %d: %s', batch_i, meta['file_name'])
pred = preprocess.annotations_inverse(pred, meta)
if args.output_directory is None:
@ -117,15 +114,16 @@ def predict(args):
file_name = os.path.basename(meta['file_name'])
output_path = os.path.join(args.output_directory, 'out_' + file_name)
print('image', batch_i, meta['file_name'], output_path)
pifpaf_out = [ann.json_data() for ann in pred]
if idx == 0:
pifpaf_outputs = pred # to only print left image for stereo
pifpaf_outs = {'left': pifpaf_out}
with open(meta_batch[0]['file_name'], 'rb') as f:
cpu_image = PIL.Image.open(f).convert('RGB')
pifpaf_outs = {
'pred': pred,
'left': [ann.json_data() for ann in pred],
'image': cpu_image}
else:
pifpaf_outs['right'] = pifpaf_out
pifpaf_outs['right'] = [ann.json_data() for ann in pred]
# 3D Predictions
if args.net in ('monoloco_pp', 'monstereo'):
@ -138,15 +136,14 @@ def predict(args):
boxes, keypoints = preprocess_pifpaf(pifpaf_outs['left'], im_size, enlarge_boxes=False)
if args.net == 'monoloco_pp':
print("Prediction with MonoLoco++")
LOG.info("Prediction with MonoLoco++")
dic_out = net.forward(keypoints, kk)
dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt, reorder=not args.social_distance)
dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt)
if args.social_distance:
show_social(args, cpu_image, output_path, pifpaf_out, dic_out)
dic_out = net.social_distance(dic_out, args)
else:
print("Prediction with MonStereo")
LOG.info("Prediction with MonStereo")
boxes_r, keypoints_r = preprocess_pifpaf(pifpaf_outs['right'], im_size)
dic_out = net.forward(keypoints, kk, keypoints_r=keypoints_r)
dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt)
@ -155,28 +152,38 @@ def predict(args):
dic_out = defaultdict(list)
kk = None
if not args.social_distance:
factory_outputs(args, annotation_painter, cpu_image, output_path, pifpaf_outputs,
dic_out=dic_out, kk=kk)
print('Image {}\n'.format(cnt) + '-' * 120)
# Outputs
factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=kk)
LOG.info('Image {}\n'.format(cnt) + '-' * 120)
cnt += 1
def factory_outputs(args, annotation_painter, cpu_image, output_path, pred, dic_out=None, kk=None):
def factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=None):
"""Output json files or images according to the choice"""
# Save json file
# Verify conflicting options
if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
assert args.net != 'pifpaf', "please use pifpaf original arguments"
if args.social_distance:
assert args.net == 'monoloco_pp', "Social distancing only works with MonoLoco++ network"
if args.net == 'pifpaf':
with openpifpaf.show.image_canvas(cpu_image, output_path) as ax:
annotation_painter.annotations(ax, pred)
annotation_painter = openpifpaf.show.AnnotationPainter()
with openpifpaf.show.image_canvas(pifpaf_outs['image'], output_path) as ax:
annotation_painter.annotations(ax, pifpaf_outs['pred'])
if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
print(output_path)
if dic_out['boxes']: # Only print in case of detections
printer = Printer(cpu_image, output_path, kk, args)
figures, axes = printer.factory_axes(dic_out)
printer.draw(figures, axes, cpu_image)
elif any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
LOG.info(output_path)
if args.social_distance:
show_social(args, pifpaf_outs['image'], output_path, pifpaf_outs['left'], dic_out)
else:
printer = Printer(pifpaf_outs['image'], output_path, kk, args)
figures, axes = printer.factory_axes(dic_out)
printer.draw(figures, axes, pifpaf_outs['image'])
if 'json' in args.output_types:
with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:
json.dump(dic_out, ff)
elif 'json' in args.output_types:
with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:
json.dump(dic_out, ff)
else:
LOG.info("No output saved, please select one among front, bird, multi, or pifpaf options")