update for experiments and readme

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 by EPFL/VITA. All rights reserved.
+Copyright 2020-2021 by EPFL/VITA. All rights reserved.
 
 This project and all its files are licensed under
 GNU AGPLv3 or later version.
@@ -6,4 +6,4 @@ GNU AGPLv3 or later version.
 If this license is not suitable for your business or project
 please contact EPFL-TTO (https://tto.epfl.ch/) for a full commercial license.
 
-This software may not be used to harm any person deliberately.
+This software may not be used to harm any person deliberately or for any military application.

docs/MonoLoco++.md

@@ -82,4 +82,53 @@ Our results are obtained with:
 
 For a more extensive list of available parameters, run:
 
-`python -m monstereo.run train --help`
+`python -m monstereo.run train --help`
+
+## Evaluation
+
+### 3D Localization
+We provide evaluation on KITTI for models trained on nuScenes or KITTI. We compare them with other monocular 
+and stereo Baselines: 
+
+[MonoLoco](TODO), 
+[Mono3D](https://www.cs.toronto.edu/~urtasun/publications/chen_etal_cvpr16.pdf), 
+[3DOP](https://xiaozhichen.github.io/papers/nips15chen.pdf), 
+[MonoDepth](https://arxiv.org/abs/1609.03677) 
+[MonoPSR](TODO) and our 
+[MonoDIS](TODO) and our 
+[Geometrical Baseline](monoloco/eval/geom_baseline.py).
+
+* **Mono3D**: download validation files from [here](http://3dimage.ee.tsinghua.edu.cn/cxz/mono3d) 
+and save them into `data/kitti/m3d`
+* **3DOP**: download validation files from [here](https://xiaozhichen.github.io/) 
+and save them into `data/kitti/3dop`
+* **MonoDepth**: compute an average depth for every instance using the following script 
+[here](https://github.com/Parrotlife/pedestrianDepth-baseline/tree/master/MonoDepth-PyTorch) 
+and save them into `data/kitti/monodepth`
+* **GeometricalBaseline**: A geometrical baseline comparison is provided. 
+
+Download the model for monoloco
+
+The average geometrical value for comparison can be obtained running:
+```
+python -m monstereo.run eval 
+--dir_ann <annotation directory> 
+--model <model path> 
+--net monoloco_pp 
+--generate
+````
+
+To include also geometric baselines and MonoLoco, add the flag ``--baselines``
+
+### Activity Estimation (Talking)
+Please follow preprocessing steps for Collective activity dataset and run pifpaf over the dataset images.
+Evaluation on this dataset is done with models trained on either KITTI or nuScenes. 
+For optimal performances, we suggest the model trained on nuScenes teaser (TODO add link)
+```
+python -m monstereo.run eval 
+--activity 
+--dataset collective
+--net monoloco_pp
+--model <path to the model>   
+--dir_ann <annotation directory>
+```

monstereo/activity.py

@@ -16,7 +16,7 @@ from matplotlib.patches import Circle, FancyArrow
 from PIL import Image
 
 from .network.process import laplace_sampling
-from .utils import open_annotations
+from .utils import open_annotations, get_task_error
 from .visuals.pifpaf_show import KeypointPainter, image_canvas
 from .network import Loco
 from .network.process import factory_for_gt, preprocess_pifpaf
@@ -37,7 +37,7 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
     if n_samples < 2:
         for idx_t in indices:
             if check_f_formations(idx, idx_t, centers, angles,
-                                  radii=radii, # Binary value
+                                  radii=radii,  # Binary value
                                   social_distance=social_distance):
                 return True
 
@@ -46,7 +46,7 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
         # Samples distance
         dds = torch.tensor(dds).view(-1, 1)
         stds = torch.tensor(stds).view(-1, 1)
-        # stds_te = get_task_error(dds)  # similar results to MonoLoco but lower true positive
+        # stds = get_task_error(dds)  # similar results to MonoLoco but lower true positive
         laplace_d = torch.cat((dds, stds), dim=1)
         samples_d = laplace_sampling(laplace_d, n_samples=n_samples)
 
@@ -72,8 +72,8 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa
 
 def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False):
     """
-    Check F-formations for people close together:
+    Check F-formations for people close together (this function do not expect far away people):
-    1) Empty space of 0.4 + meters (no other people or themselves inside)
+    1) Empty space of a certain radius (no other people or themselves inside)
     2) People looking inward
     """
 

monstereo/eval/eval_activity.py

@@ -32,10 +32,10 @@ class ActivityEvaluator:
             self.sequences = ['seq02', 'seq14', 'seq12', 'seq13', 'seq11', 'seq36']
             # folders_collective = ['seq02']
             self.dir_data = 'data/activity/dataset'
-            self.THRESHOLD_PROB = 0.25  # Concordance for samples
+            self.THRESHOLD_PROB = 0.2  # Concordance for samples
             self.THRESHOLD_DIST = 2  # Threshold to check distance of people
             self.RADII = (0.3, 0.5)  # expected radii of the o-space
-            self.PIFPAF_CONF = 0.4
+            self.PIFPAF_CONF = 0.3
             self.SOCIAL_DISTANCE = False
         # -------------------------------------------------------------------------------------------------------------
 
@@ -76,7 +76,7 @@ class ActivityEvaluator:
 
                 # Collect PifPaf files and calibration
                 basename = os.path.basename(im_path)
-                extension = '.pifpaf.json'
+                extension = '.predictions.json'
                 path_pif = os.path.join(self.dir_ann, basename + extension)
                 annotations = open_annotations(path_pif)
                 kk, _ = factory_for_gt(im_size, verbose=False)

monstereo/eval/generate_kitti.py

@@ -241,6 +241,7 @@ def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco', ca
                 conf_scale = 0.03
             elif mode == 'monoloco_pp':
                 conf_scale = 0.033
+                # conf_scale = 0.036 nuScenes for having same recall
             else:
                 conf_scale = 0.055
             conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2))

monstereo/prep/preprocess_nu.py

@@ -87,7 +87,7 @@ class PreprocessNuscenes:
             while not current_token == "":
                 sample_dic = self.nusc.get('sample', current_token)
                 cnt_samples += 1
-
+                # if (cnt_samples % 4 == 0) and (cnt_ann < 3000):
                 # Extract all the sample_data tokens for each sample
                 for cam in self.CAMERAS:
                     sd_token = sample_dic['data'][cam]
@@ -105,7 +105,7 @@ class PreprocessNuscenes:
                     self.dic_names[basename + '.jpg']['K'] = copy.deepcopy(kk)
 
                     # Run IoU with pifpaf detections and save
-                    path_pif = os.path.join(self.dir_ann, name + '.pifpaf.json')
+                    path_pif = os.path.join(self.dir_ann, name + '.predictions.json')
                     exists = os.path.isfile(path_pif)
 
                     if exists:
@@ -114,7 +114,6 @@ class PreprocessNuscenes:
                             boxes, keypoints = preprocess_pifpaf(annotations, im_size=(1600, 900))
                     else:
                         continue
-
                     if keypoints:
                         matches = get_iou_matches(boxes, boxes_gt, self.iou_min)
                         for (idx, idx_gt) in matches:
@@ -130,7 +129,6 @@ class PreprocessNuscenes:
                             append_cluster(self.dic_jo, phase, inp, lab, keypoint)
                             cnt_ann += 1
                             sys.stdout.write('\r' + 'Saved annotations {}'.format(cnt_ann) + '\t')
-
                 current_token = sample_dic['next']
 
         with open(os.path.join(self.path_joints), 'w') as f:
@@ -139,7 +137,7 @@ class PreprocessNuscenes:
             json.dump(self.dic_names, f)
         end = time.time()
 
-        extract_box_average(self.dic_jo['train']['boxes_3d'])
+        # extract_box_average(self.dic_jo['train']['boxes_3d'])
         print("\nSaved {} annotations for {} samples in {} scenes. Total time: {:.1f} minutes"
               .format(cnt_ann, cnt_samples, cnt_scenes, (end-start)/60))
         print("\nOutput files:\n{}\n{}\n".format(self.path_names, self.path_joints))

setup.py

@@ -27,9 +27,7 @@ setup(
     zip_safe=False,
 
     install_requires=[
-        'openpifpaf==0.8.0',
+        'openpifpaf>=0.11'
-        'torch==1.1.0',
-        'torchvision==0.3.0'
     ],
     extras_require={
         'eval': [