diff --git a/LICENSE b/LICENSE index 40f5c41..8ddfc99 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020 by EPFL/VITA. All rights reserved. +Copyright 2020-2021 by EPFL/VITA. All rights reserved. This project and all its files are licensed under GNU AGPLv3 or later version. @@ -6,4 +6,4 @@ GNU AGPLv3 or later version. If this license is not suitable for your business or project please contact EPFL-TTO (https://tto.epfl.ch/) for a full commercial license. -This software may not be used to harm any person deliberately. +This software may not be used to harm any person deliberately or for any military application. diff --git a/docs/MonoLoco++.md b/docs/MonoLoco++.md index a29bbbf..febb276 100644 --- a/docs/MonoLoco++.md +++ b/docs/MonoLoco++.md @@ -82,4 +82,53 @@ Our results are obtained with: For a more extensive list of available parameters, run: -`python -m monstereo.run train --help` \ No newline at end of file +`python -m monstereo.run train --help` + +## Evaluation + +### 3D Localization +We provide evaluation on KITTI for models trained on nuScenes or KITTI. We compare them with other monocular +and stereo Baselines: + +[MonoLoco](TODO), +[Mono3D](https://www.cs.toronto.edu/~urtasun/publications/chen_etal_cvpr16.pdf), +[3DOP](https://xiaozhichen.github.io/papers/nips15chen.pdf), +[MonoDepth](https://arxiv.org/abs/1609.03677) +[MonoPSR](TODO) and our +[MonoDIS](TODO) and our +[Geometrical Baseline](monoloco/eval/geom_baseline.py). + +* **Mono3D**: download validation files from [here](http://3dimage.ee.tsinghua.edu.cn/cxz/mono3d) +and save them into `data/kitti/m3d` +* **3DOP**: download validation files from [here](https://xiaozhichen.github.io/) +and save them into `data/kitti/3dop` +* **MonoDepth**: compute an average depth for every instance using the following script +[here](https://github.com/Parrotlife/pedestrianDepth-baseline/tree/master/MonoDepth-PyTorch) +and save them into `data/kitti/monodepth` +* **GeometricalBaseline**: A geometrical baseline comparison is provided. + +Download the model for monoloco + +The average geometrical value for comparison can be obtained running: +``` +python -m monstereo.run eval +--dir_ann +--model +--net monoloco_pp +--generate +```` + +To include also geometric baselines and MonoLoco, add the flag ``--baselines`` + +### Activity Estimation (Talking) +Please follow preprocessing steps for Collective activity dataset and run pifpaf over the dataset images. +Evaluation on this dataset is done with models trained on either KITTI or nuScenes. +For optimal performances, we suggest the model trained on nuScenes teaser (TODO add link) +``` +python -m monstereo.run eval +--activity +--dataset collective +--net monoloco_pp +--model +--dir_ann +``` \ No newline at end of file diff --git a/monstereo/activity.py b/monstereo/activity.py index 0cbf395..aec94ed 100644 --- a/monstereo/activity.py +++ b/monstereo/activity.py @@ -16,7 +16,7 @@ from matplotlib.patches import Circle, FancyArrow from PIL import Image from .network.process import laplace_sampling -from .utils import open_annotations +from .utils import open_annotations, get_task_error from .visuals.pifpaf_show import KeypointPainter, image_canvas from .network import Loco from .network.process import factory_for_gt, preprocess_pifpaf @@ -37,7 +37,7 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa if n_samples < 2: for idx_t in indices: if check_f_formations(idx, idx_t, centers, angles, - radii=radii, # Binary value + radii=radii, # Binary value social_distance=social_distance): return True @@ -46,7 +46,7 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa # Samples distance dds = torch.tensor(dds).view(-1, 1) stds = torch.tensor(stds).view(-1, 1) - # stds_te = get_task_error(dds) # similar results to MonoLoco but lower true positive + # stds = get_task_error(dds) # similar results to MonoLoco but lower true positive laplace_d = torch.cat((dds, stds), dim=1) samples_d = laplace_sampling(laplace_d, n_samples=n_samples) @@ -72,8 +72,8 @@ def social_interactions(idx, centers, angles, dds, stds=None, social_distance=Fa def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False): """ - Check F-formations for people close together: - 1) Empty space of 0.4 + meters (no other people or themselves inside) + Check F-formations for people close together (this function do not expect far away people): + 1) Empty space of a certain radius (no other people or themselves inside) 2) People looking inward """ diff --git a/monstereo/eval/eval_activity.py b/monstereo/eval/eval_activity.py index f74ba09..b485cb9 100644 --- a/monstereo/eval/eval_activity.py +++ b/monstereo/eval/eval_activity.py @@ -32,10 +32,10 @@ class ActivityEvaluator: self.sequences = ['seq02', 'seq14', 'seq12', 'seq13', 'seq11', 'seq36'] # folders_collective = ['seq02'] self.dir_data = 'data/activity/dataset' - self.THRESHOLD_PROB = 0.25 # Concordance for samples + self.THRESHOLD_PROB = 0.2 # Concordance for samples self.THRESHOLD_DIST = 2 # Threshold to check distance of people self.RADII = (0.3, 0.5) # expected radii of the o-space - self.PIFPAF_CONF = 0.4 + self.PIFPAF_CONF = 0.3 self.SOCIAL_DISTANCE = False # ------------------------------------------------------------------------------------------------------------- @@ -76,7 +76,7 @@ class ActivityEvaluator: # Collect PifPaf files and calibration basename = os.path.basename(im_path) - extension = '.pifpaf.json' + extension = '.predictions.json' path_pif = os.path.join(self.dir_ann, basename + extension) annotations = open_annotations(path_pif) kk, _ = factory_for_gt(im_size, verbose=False) diff --git a/monstereo/eval/generate_kitti.py b/monstereo/eval/generate_kitti.py index 9113da1..4be8618 100644 --- a/monstereo/eval/generate_kitti.py +++ b/monstereo/eval/generate_kitti.py @@ -241,6 +241,7 @@ def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco', ca conf_scale = 0.03 elif mode == 'monoloco_pp': conf_scale = 0.033 + # conf_scale = 0.036 nuScenes for having same recall else: conf_scale = 0.055 conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2)) diff --git a/monstereo/prep/preprocess_nu.py b/monstereo/prep/preprocess_nu.py index b27fa78..4bb8caa 100644 --- a/monstereo/prep/preprocess_nu.py +++ b/monstereo/prep/preprocess_nu.py @@ -87,7 +87,7 @@ class PreprocessNuscenes: while not current_token == "": sample_dic = self.nusc.get('sample', current_token) cnt_samples += 1 - + # if (cnt_samples % 4 == 0) and (cnt_ann < 3000): # Extract all the sample_data tokens for each sample for cam in self.CAMERAS: sd_token = sample_dic['data'][cam] @@ -105,7 +105,7 @@ class PreprocessNuscenes: self.dic_names[basename + '.jpg']['K'] = copy.deepcopy(kk) # Run IoU with pifpaf detections and save - path_pif = os.path.join(self.dir_ann, name + '.pifpaf.json') + path_pif = os.path.join(self.dir_ann, name + '.predictions.json') exists = os.path.isfile(path_pif) if exists: @@ -114,7 +114,6 @@ class PreprocessNuscenes: boxes, keypoints = preprocess_pifpaf(annotations, im_size=(1600, 900)) else: continue - if keypoints: matches = get_iou_matches(boxes, boxes_gt, self.iou_min) for (idx, idx_gt) in matches: @@ -130,7 +129,6 @@ class PreprocessNuscenes: append_cluster(self.dic_jo, phase, inp, lab, keypoint) cnt_ann += 1 sys.stdout.write('\r' + 'Saved annotations {}'.format(cnt_ann) + '\t') - current_token = sample_dic['next'] with open(os.path.join(self.path_joints), 'w') as f: @@ -139,7 +137,7 @@ class PreprocessNuscenes: json.dump(self.dic_names, f) end = time.time() - extract_box_average(self.dic_jo['train']['boxes_3d']) + # extract_box_average(self.dic_jo['train']['boxes_3d']) print("\nSaved {} annotations for {} samples in {} scenes. Total time: {:.1f} minutes" .format(cnt_ann, cnt_samples, cnt_scenes, (end-start)/60)) print("\nOutput files:\n{}\n{}\n".format(self.path_names, self.path_joints)) diff --git a/setup.py b/setup.py index 6771aa4..4a724e7 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,7 @@ setup( zip_safe=False, install_requires=[ - 'openpifpaf==0.8.0', - 'torch==1.1.0', - 'torchvision==0.3.0' + 'openpifpaf>=0.11' ], extras_require={ 'eval': [