Camera parameters (#5)

* change reorder flag

* make social distance as separate function

* temp

* temp

* refactor names pifpaf outputs

* verify conflicting options

* add logging

* custom camera parameters

* convert back to print

* add pyc files

.gitignore

@@ -2,7 +2,7 @@
 data/
 .DS_store
 __pycache__
-Monoloco/*.pyc
+monstereo/*.pyc
 .pytest*
 build/
 dist/

monstereo/__init__.py

@@ -1,4 +1,4 @@
 
 """Open implementation of MonoLoco++ / MonStereo."""
 
-__version__ = '0.2.0'
+__version__ = '0.2.1'

monstereo/activity.py

@@ -111,18 +111,11 @@ def show_social(args, image_t, output_path, annotations, dic_out):
     assert 'front' in args.output_types or 'bird' in args.output_types, "outputs allowed: front and/or bird"
 
     angles = dic_out['angles']
-    dds = dic_out['dds_pred']
     stds = dic_out['stds_ale']
     xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
 
     # Prepare color for social distancing
-    colors = ['r' if social_interactions(idx, xz_centers, angles, dds,
+    colors = ['r' if flag else 'deepskyblue' for flag in dic_out['social_distance']]
-                                         stds=stds,
-                                         threshold_prob=args.threshold_prob,
-                                         threshold_dist=args.threshold_dist,
-                                         radii=args.radii)
-              else 'deepskyblue'
-              for idx, _ in enumerate(dic_out['xyz_pred'])]
 
     # Draw keypoints and orientation
     if 'front' in args.output_types:

monstereo/network/net.py

@@ -14,6 +14,7 @@ import torch
 from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance
 from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\
     filter_outputs, cluster_outputs, unnormalize_bi
+from ..activity import social_interactions
 from .architectures import MonolocoModel, MonStereoModel
 
 
@@ -172,7 +173,7 @@ class Loco:
             if verbose:
                 print("NO ground-truth associated")
 
-        if reorder:
+        if reorder and matches:
             matches = reorder_matches(matches, boxes, mode='left_right')
 
         all_idxs = [idx for idx, _ in matches] + not_matches
@@ -233,6 +234,23 @@ class Loco:
             dic_out['xyz_real'].append(xyz_real.squeeze().tolist())
         return dic_out
 
+    @staticmethod
+    def social_distance(dic_out, args):
+
+        angles = dic_out['angles']
+        dds = dic_out['dds_pred']
+        stds = dic_out['stds_ale']
+        xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
+
+        # Prepare color for social distancing
+        dic_out['social_distance'] = [bool(social_interactions(idx, xz_centers, angles, dds,
+                                                               stds=stds,
+                                                               threshold_prob=args.threshold_prob,
+                                                               threshold_dist=args.threshold_dist,
+                                                               radii=args.radii))
+                                      for idx, _ in enumerate(dic_out['xyz_pred'])]
+        return dic_out
+
 
 def median_disparity(dic_out, keypoints, keypoints_r, mask):
     """

monstereo/network/process.py

@@ -1,11 +1,15 @@
 
 import json
 import os
+import logging
 
 import numpy as np
 import torch
 import torchvision
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 from ..utils import get_keypoints, pixel_to_camera, to_cartesian, back_correct_angles
 
 BF = 0.54 * 721
@@ -13,6 +17,9 @@ z_min = 4
 z_max = 60
 D_MIN = BF / z_max
 D_MAX = BF / z_min
+FL = 5.7  # nuScenes focal length (mm)
+Sx = 7.2  # nuScenes sensor size (mm)
+Sy = 5.4  # nuScenes sensor size (mm)
 
 
 def preprocess_monstereo(keypoints, keypoints_r, kk):
@@ -67,31 +74,28 @@ def factory_for_gt(im_size, name=None, path_gt=None, verbose=True):
         with open(path_gt, 'r') as f:
             dic_names = json.load(f)
         if verbose:
-            print('-' * 120 + "\nGround-truth file opened")
+            logger.info('-' * 120 + "\nGround-truth file opened")
     except (FileNotFoundError, TypeError):
         if verbose:
-            print('-' * 120 + "\nGround-truth file not found")
+            logger.info('-' * 120 + "\nGround-truth file not found")
         dic_names = {}
 
     try:
         kk = dic_names[name]['K']
         dic_gt = dic_names[name]
         if verbose:
-            print("Matched ground-truth file!")
+            logger.info("Matched ground-truth file!")
     except KeyError:
         dic_gt = None
-        x_factor = im_size[0] / 1600
+        if im_size[0] / im_size[1] > 2.5:  # KITTI default
-        y_factor = im_size[1] / 900
-        pixel_factor = (x_factor + y_factor) / 1.75  # 1.75 for MOT
-        # pixel_factor = 1
-        if im_size[0] / im_size[1] > 2.5:
             kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]]  # Kitti calibration
-        else:
+        else:  # nuScenes camera parameters
-            kk = [[1266.4 * pixel_factor, 0., 816.27 * x_factor],
+            kk = [
-                  [0, 1266.4 * pixel_factor, 491.5 * y_factor],
+                [im_size[0]*FL/Sx, 0., im_size[0]/2],
-                  [0., 0., 1.]]  # nuScenes calibration
+                [0., im_size[1]*FL/Sy, im_size[1]/2],
+                [0., 0., 1.]]
         if verbose:
-            print("Using a standard calibration matrix...")
+            logger.info("Using a standard calibration matrix...")
 
     return kk, dic_gt
 

monstereo/predict.py

@@ -27,6 +27,7 @@ LOG = logging.getLogger(__name__)
 
 OPENPIFPAF_PATH = 'data/models/shufflenetv2k30-201104-224654-cocokp-d75ed641.pkl'  # Default model
 
+
 def factory_from_args(args):
 
     # Data
@@ -40,7 +41,7 @@ def factory_from_args(args):
         if os.path.exists(OPENPIFPAF_PATH):
             args.checkpoint = OPENPIFPAF_PATH
         else:
-            print("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. "
+            LOG.info("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. "
                   "Using a ShuffleNet backbone")
             args.checkpoint = 'shufflenetv2k30'
 
@@ -64,7 +65,7 @@ def factory_from_args(args):
 
     # Make default pifpaf argument
     args.force_complete_pose = True
-    print("Force complete pose is active")
+    LOG.info("Force complete pose is active")
 
     # Configure
     decoder.configure(args)
@@ -82,7 +83,6 @@ def predict(args):
 
     # Load Models
     assert args.net in ('monoloco_pp', 'monstereo', 'pifpaf')
-
     if args.net in ('monoloco_pp', 'monstereo'):
         net = Loco(model=args.model, net=args.net, device=args.device, n_dropout=args.n_dropout, p_dropout=args.dropout)
 
@@ -99,15 +99,12 @@ def predict(args):
         data, batch_size=args.batch_size, shuffle=False,
         pin_memory=False, collate_fn=datasets.collate_images_anns_meta)
 
-    # visualizers
-    annotation_painter = openpifpaf.show.AnnotationPainter()
-
     for batch_i, (image_tensors_batch, _, meta_batch) in enumerate(data_loader):
         pred_batch = processor.batch(model, image_tensors_batch, device=args.device)
 
         # unbatch (only for MonStereo)
         for idx, (pred, meta) in enumerate(zip(pred_batch, meta_batch)):
-            LOG.info('batch %d: %s', batch_i, meta['file_name'])
+            print('batch %d: %s', batch_i, meta['file_name'])
             pred = preprocess.annotations_inverse(pred, meta)
 
             if args.output_directory is None:
@@ -117,15 +114,16 @@ def predict(args):
                 file_name = os.path.basename(meta['file_name'])
                 output_path = os.path.join(args.output_directory, 'out_' + file_name)
             print('image', batch_i, meta['file_name'], output_path)
-            pifpaf_out = [ann.json_data() for ann in pred]
 
             if idx == 0:
-                pifpaf_outputs = pred  # to only print left image for stereo
-                pifpaf_outs = {'left': pifpaf_out}
                 with open(meta_batch[0]['file_name'], 'rb') as f:
                     cpu_image = PIL.Image.open(f).convert('RGB')
+                pifpaf_outs = {
+                    'pred': pred,
+                    'left': [ann.json_data() for ann in pred],
+                    'image': cpu_image}
             else:
-                pifpaf_outs['right'] = pifpaf_out
+                pifpaf_outs['right'] = [ann.json_data() for ann in pred]
 
         # 3D Predictions
         if args.net in ('monoloco_pp', 'monstereo'):
@@ -138,15 +136,14 @@ def predict(args):
             boxes, keypoints = preprocess_pifpaf(pifpaf_outs['left'], im_size, enlarge_boxes=False)
 
             if args.net == 'monoloco_pp':
-                print("Prediction with MonoLoco++")
+                LOG.info("Prediction with MonoLoco++")
                 dic_out = net.forward(keypoints, kk)
-                dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt, reorder=not args.social_distance)
+                dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt)
-
                 if args.social_distance:
-                    show_social(args, cpu_image, output_path, pifpaf_out, dic_out)
+                    dic_out = net.social_distance(dic_out, args)
 
             else:
-                print("Prediction with MonStereo")
+                LOG.info("Prediction with MonStereo")
                 boxes_r, keypoints_r = preprocess_pifpaf(pifpaf_outs['right'], im_size)
                 dic_out = net.forward(keypoints, kk, keypoints_r=keypoints_r)
                 dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt)
@@ -155,28 +152,38 @@ def predict(args):
             dic_out = defaultdict(list)
             kk = None
 
-        if not args.social_distance:
+        # Outputs
-            factory_outputs(args, annotation_painter, cpu_image, output_path, pifpaf_outputs,
+        factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=kk)
-                            dic_out=dic_out, kk=kk)
+        LOG.info('Image {}\n'.format(cnt) + '-' * 120)
-        print('Image {}\n'.format(cnt) + '-' * 120)
         cnt += 1
 
 
-def factory_outputs(args, annotation_painter, cpu_image, output_path, pred, dic_out=None, kk=None):
+def factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=None):
     """Output json files or images according to the choice"""
 
-    # Save json file
+    # Verify conflicting options
+    if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
+        assert args.net != 'pifpaf', "please use pifpaf original arguments"
+        if args.social_distance:
+            assert args.net == 'monoloco_pp', "Social distancing only works with MonoLoco++ network"
+
     if args.net == 'pifpaf':
-        with openpifpaf.show.image_canvas(cpu_image, output_path) as ax:
+        annotation_painter = openpifpaf.show.AnnotationPainter()
-            annotation_painter.annotations(ax, pred)
+        with openpifpaf.show.image_canvas(pifpaf_outs['image'], output_path) as ax:
+            annotation_painter.annotations(ax, pifpaf_outs['pred'])
 
-        if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
+    elif any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
-            print(output_path)
+        LOG.info(output_path)
-            if dic_out['boxes']:  # Only print in case of detections
+        if args.social_distance:
-                printer = Printer(cpu_image, output_path, kk, args)
+            show_social(args, pifpaf_outs['image'], output_path, pifpaf_outs['left'], dic_out)
-                figures, axes = printer.factory_axes(dic_out)
+        else:
-                printer.draw(figures, axes, cpu_image)
+            printer = Printer(pifpaf_outs['image'], output_path, kk, args)
+            figures, axes = printer.factory_axes(dic_out)
+            printer.draw(figures, axes, pifpaf_outs['image'])
 
-        if 'json' in args.output_types:
+    elif 'json' in args.output_types:
-            with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:
+        with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:
-                json.dump(dic_out, ff)
+            json.dump(dic_out, ff)
+
+    else:
+        LOG.info("No output saved, please select one among front, bird, multi, or pifpaf options")