diff --git a/README.md b/README.md index 89496cd..efc861a 100644 --- a/README.md +++ b/README.md @@ -112,14 +112,16 @@ data/kitti/images` Download nuScenes dataset (any version: Mini, Teaser or TrainVal) from [nuScenes](https://www.nuscenes.org/download), save it anywhere and soft link it in `data/nuscenes` -### 3) Train Val Splits -Fo -### Input joints for training -MonoLoco is trained using 2D human pose joints detected by pifpaf and matched with the ground truth location provided by +### Annotations to preprocess +MonoLoco is trained using 2D human pose joints. To create them run pifaf over KITTI or nuScenes training images. +You can create them running the predict script and using `--network pifpaf`. + +### Inputs joints for training +MonoLoco is trained using 2D human pose joints matched with the ground truth location provided by nuScenes or KITTI Dataset. To create the joints run: `python src/main.py prep` specifying: -1. `--dir_ann` annotation directory containing pifpaf joints. You can create them running the predict script and using -`--network pifpaf`. +1. `--dir_ann` annotation directory containing pifpaf joints of kitti or nuScenes. + 2. `--dataset` Which dataset to preprocess. For nuscenes, all three versions of the dataset are supported: nuscenes_mini, nuscenes, nuscenes_teaser. @@ -129,11 +131,7 @@ by the image name to easily access ground truth files for evaluation and predict - - - - -### Train +# Train Provide the json file containing the preprocess joints (**joints.json**) as argument. As simple as `python3 src/main.py --train --joints 'data/arrays/joints.json` @@ -144,16 +142,37 @@ All the hyperparameters options can be checked at `python3 src/main.py train --h Random search in log space is provided. An example: `python3 src/main.py train --hyp --multiplier 10 --r_seed 1`. One iteration of the multiplier includes 6 runs. -# Eval -Evaluate performances of the trained model on KITTI or Nuscenes Dataset. Compare them with other monocular + +# Evaluation +Evaluate performances of the trained model on KITTI or Nuscenes Dataset. +### 1) nuScenes +Evaluation on nuScenes is already provided during training. It is also possible to evaluate an existing model running +`python src/main.py eval --dataset nuscenes --model ` + +### 2) KITTI +### Baselines +We provide evaluation on KITTI for models trained on nuScenes or KITTI. We compare them with other monocular and stereo Baselines: -[Mono3D](http://3dimage.ee.tsinghua.edu.cn/cxz/mono3d), + +[Mono3D](https://www.cs.toronto.edu/~urtasun/publications/chen_etal_cvpr16.pdf), [3DOP](https://xiaozhichen.github.io/papers/nips15chen.pdf), [MonoDepth](https://arxiv.org/abs/1609.03677) and our [Geometrical Baseline](src/eval/geom_baseline.py). -Alternatively we provide the links to download them. +* **Mono3D**: download validation files from [here](http://3dimage.ee.tsinghua.edu.cn/cxz/mono3d) +and save them into `data/kitti/m3d` +* **3DOP**: download validation files from [here](https://xiaozhichen.github.io/) +and save them into `data/kitti/3dop` +* **MonoDepth**: compute an average depth for every instance using the following script +[here](https://github.com/Parrotlife/pedestrianDepth-baseline/tree/master/MonoDepth-PyTorch) +and save them into `data/kitti/monodepth` +* **GeometricalBaseline**: A geometrical baseline comparison is provided. +The best average value for comparison can be created running `python src/main.py eval --geometric` +#### Evaluation +First the model preprocess the joints starting from json annotations predicted from pifpaf, runs the model and save the results +in txt file with format comparable to other baseline. +Then the model performs evaluation. The following graph is obtained running: `python3 src/main.py eval --dataset kitti --model data/models/base_model.pickle` diff --git a/splits/splt_nuscenes_teaser.json b/splits/split_nuscenes_teaser.json similarity index 100% rename from splits/splt_nuscenes_teaser.json rename to splits/split_nuscenes_teaser.json diff --git a/src/eval/kitti_eval.py b/src/eval/kitti_eval.py index 974f9b3..101cef7 100644 --- a/src/eval/kitti_eval.py +++ b/src/eval/kitti_eval.py @@ -36,7 +36,7 @@ class KittiEval: self.dir_md = os.path.join('data', 'kitti', 'monodepth') self.dir_psm = os.path.join('data', 'kitti', 'psm') self.dir_our = os.path.join('data', 'kitti', 'monoloco') - path_val = os.path.join('data', 'kitti', 'val.txt') + path_val = os.path.join('splits', 'kitti_val.txt') dir_logs = os.path.join('data', 'logs') assert dir_logs, "No directory to save final statistics" diff --git a/src/eval/run_kitti.py b/src/eval/run_kitti.py index e71955d..e17fb59 100644 --- a/src/eval/run_kitti.py +++ b/src/eval/run_kitti.py @@ -35,7 +35,7 @@ class RunKitti: # Load the model input_size = 17 * 2 use_cuda = torch.cuda.is_available() - self.device = torch.device("cuda:0" if use_cuda else "cpu") + self.device = torch.device("cuda" if use_cuda else "cpu") self.model = LinearModel(input_size=input_size, output_size=2, linear_size=hidden_size, p_dropout=dropout, num_stage=n_stage) self.model.load_state_dict(torch.load(model, map_location=lambda storage, loc: storage)) diff --git a/src/features/preprocess_ki.py b/src/features/preprocess_ki.py index 3016562..8e67d7b 100644 --- a/src/features/preprocess_ki.py +++ b/src/features/preprocess_ki.py @@ -109,7 +109,7 @@ class PreprocessKitti: dds.append(dd) self.dic_names[basename + '.png']['boxes'].append(box) self.dic_names[basename + '.png']['dds'].append(dd) - self.dic_names[basename + '.png']['K'].append(kk) + self.dic_names[basename + '.png']['K'].append(kk.tolist()) self.cnt_gt += 1 # Find the annotations if exists @@ -131,7 +131,7 @@ class PreprocessKitti: self.dic_jo[phase]['kps'].append(uv_kps[ii]) self.dic_jo[phase]['X'].append(inputs[ii]) self.dic_jo[phase]['Y'].append([dds[idx_max]]) # Trick to make it (nn,1) - self.dic_jo[phase]['K'].append(kk) + self.dic_jo[phase]['K'].append(kk.tolist()) self.dic_jo[phase]['names'].append(name) # One image name for each annotation self.append_cluster(self.dic_jo, phase, inputs[ii], dds[idx_max], uv_kps[ii]) self.dic_cnt[phase] += 1 diff --git a/src/features/preprocess_nu.py b/src/features/preprocess_nu.py index 8da959c..9442f36 100644 --- a/src/features/preprocess_nu.py +++ b/src/features/preprocess_nu.py @@ -77,7 +77,7 @@ class PreprocessNuscenes: teaser_scenes = ff.read().splitlines() self.scenes = self.nusc.scene self.scenes = [scene for scene in self.scenes if scene['token'] in teaser_scenes] - with open("data/splits/split_nuscenes_teaser.json", "r") as ff: + with open("splits/split_nuscenes_teaser.json", "r") as ff: dic_split = json.load(ff) self.split_train = [scene['name'] for scene in self.scenes if scene['token'] in dic_split['train']] self.split_val = [scene['name'] for scene in self.scenes if scene['token'] in dic_split['val']] diff --git a/src/main.py b/src/main.py index b227986..923978f 100644 --- a/src/main.py +++ b/src/main.py @@ -36,7 +36,7 @@ def cli(): prep_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints', default='/data/lorenzo-data/nuscenes_new/annotations/scale_1_april_new') prep_parser.add_argument('--dir_nuscenes', help='directory of nuscenes devkit', - default='/data/lorenzo-data/nuscenes_new/') + default='data/nuscenes/') # Predict (2D pose and/or 3D location from images) # 0) General arguments @@ -92,11 +92,10 @@ def cli(): eval_parser.add_argument('--dataset', help='datasets to evaluate, kitti or nuscenes', default='kitti') eval_parser.add_argument('--geometric', help='to evaluate geometric distance', action='store_true') eval_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints', - default='/data/lorenzo-data/kitti/annotations/pif_factor_2_april') - eval_parser.add_argument('--model', help='path of MonoLoco model to load', - default="data/models/monoloco-190507-0943.pkl") + default='/data/lorenzo-data/kitti/annotations/scale_2_april') + eval_parser.add_argument('--model', help='path of MonoLoco model to load') eval_parser.add_argument('--joints', help='Json file with input joints to evaluate (for nuscenes)', - default='data/arrays/joints-nuscenes-190508-1102.json') + default='data/arrays/joints-nuscenes_teaser-190513-1423.json') eval_parser.add_argument('--n_dropout', type=int, help='Epistemic uncertainty evaluation', default=0) eval_parser.add_argument('--run_kitti', help='create txt files for validation of kitti', action='store_true') eval_parser.add_argument('--dropout', type=float, help='dropout. Default no dropout', default=0.2)