diff --git a/src/eval/geom_baseline.py b/src/eval/geom_baseline.py index 6f1dcf1..d49295e 100644 --- a/src/eval/geom_baseline.py +++ b/src/eval/geom_baseline.py @@ -44,15 +44,14 @@ class GeomBaseline: """ cnt_tot = 0 + dic_dist = defaultdict(lambda: defaultdict(list)) + # Access the joints file with open(self.joints, 'r') as ff: dic_joints = json.load(ff) - dic_dist = defaultdict(lambda: defaultdict(list)) - # Calculate distances for all the segments for phase in ['train', 'val']: - cnt = update_distances(dic_joints[phase], dic_dist, phase, self.average_y) cnt_tot += cnt diff --git a/src/eval/kitti_eval.py b/src/eval/kitti_eval.py index 9436787..f2ca67b 100644 --- a/src/eval/kitti_eval.py +++ b/src/eval/kitti_eval.py @@ -4,9 +4,9 @@ import os import math import logging from collections import defaultdict -import copy import datetime -from utils.misc import get_idx_max + +from utils.misc import get_iou_matches from utils.kitti import check_conditions, get_category, split_training, parse_ground_truth from visuals.results import print_results @@ -44,7 +44,8 @@ class KittiEval: assert os.path.exists(self.dir_m3d) and os.path.exists(self.dir_our) \ and os.path.exists(self.dir_3dop) - self.dic_thresh_iou = {'m3d': thresh_iou_m3d, '3dop': thresh_iou_m3d, 'md': thresh_iou_our, 'our': thresh_iou_our} + self.dic_thresh_iou = {'m3d': thresh_iou_m3d, '3dop': thresh_iou_m3d, + 'md': thresh_iou_our, 'our': thresh_iou_our} self.dic_thresh_conf = {'m3d': thresh_conf_m3d, '3dop': thresh_conf_m3d, 'our': thresh_conf_our} # Extract validation images for evaluation @@ -64,27 +65,24 @@ class KittiEval: path_md = os.path.join(self.dir_md, name) # Iterate over each line of the gt file and save box location and distances - boxes_gt, dds_gt, truncs_gt, occs_gt = parse_ground_truth(path_gt) - cnt_gt += len(boxes_gt) + out_gt = parse_ground_truth(path_gt) + cnt_gt += len(out_gt[0]) # Extract annotations for the same file - if boxes_gt: - boxes_m3d, dds_m3d = self._parse_txts(path_m3d, method='m3d') - boxes_3dop, dds_3dop = self._parse_txts(path_3dop, method='3dop') - boxes_md, dds_md = self._parse_txts(path_md, method='md') - boxes_our, dds_our, stds_ale, stds_epi, _, dds_geom, _, _ = \ - self._parse_txts(path_our, method='our') + if out_gt[0]: + out_m3d = self._parse_txts(path_m3d, method='m3d') + out_3dop = self._parse_txts(path_3dop, method='3dop') + out_md = self._parse_txts(path_md, method='md') + out_our = self._parse_txts(path_our, method='our') # Compute the error with ground truth - self._estimate_error_base(boxes_m3d, dds_m3d, boxes_gt, dds_gt, truncs_gt, occs_gt, method='m3d') - self._estimate_error_base(boxes_3dop, dds_3dop, boxes_gt, dds_gt, truncs_gt, occs_gt, method='3dop') - self._estimate_error_base(boxes_md, dds_md, boxes_gt, dds_gt, truncs_gt, occs_gt, method='md') - self._estimate_error_mloco(boxes_our, dds_our, stds_ale, stds_epi, dds_geom, - boxes_gt, dds_gt, truncs_gt, occs_gt) + self._estimate_error(out_gt, out_m3d, method='m3d') + self._estimate_error(out_gt, out_3dop, method='3dop') + self._estimate_error(out_gt, out_md, method='md') + self._estimate_error(out_gt, out_our, method='our') # Iterate over all the files together to find a pool of common annotations - self._compare_error(boxes_m3d, dds_m3d, boxes_3dop, dds_3dop, boxes_md, dds_md, boxes_our, dds_our, - boxes_gt, dds_gt, truncs_gt, occs_gt, dds_geom) + self._compare_error(out_gt, out_m3d, out_3dop, out_md, out_our) # Update statistics of errors and uncertainty for key in self.errors: @@ -128,8 +126,8 @@ class KittiEval: stds_ale = [] stds_epi = [] dds_geom = [] - xyzs = [] - xy_kps = [] + # xyzs = [] + # xy_kps = [] # Iterate over each line of the txt file if method in ['3dop', 'm3d']: @@ -166,26 +164,6 @@ class KittiEval: except FileNotFoundError: return [], [] - elif method == 'psm': - try: - with open(path, "r") as ff: - for line in ff: - box = [float(x[:-1]) for x in line[1:-1].split(',')[0:4]] - delta_h = (box[3] - box[1]) / 10 - delta_w = (box[2] - box[0]) / 10 - assert delta_h > 0 and delta_w > 0, "Bounding box <=0" - box[0] -= delta_w - box[1] -= delta_h - box[2] += delta_w - box[3] += delta_h - boxes.append(box) - dds.append(float(line.split()[5][:-1])) - self.dic_cnt[method] += 1 - return boxes, dds - - except FileNotFoundError: - return [], [] - else: assert method == 'our', "method not recognized" try: @@ -195,116 +173,69 @@ class KittiEval: line_list = [float(x) for x in line_our.split()] if check_conditions(line_list, thresh=self.dic_thresh_conf[method], mode=method): boxes.append(line_list[:4]) - xyzs.append(line_list[4:7]) + # xyzs.append(line_list[4:7]) dds.append(line_list[7]) stds_ale.append(line_list[8]) stds_epi.append(line_list[9]) dds_geom.append(line_list[11]) - xy_kps.append(line_list[12:]) + # xy_kps.append(line_list[12:]) self.dic_cnt[method] += 1 - kk_list = [float(x) for x in file_lines[-1].split()] + # kk_list = [float(x) for x in file_lines[-1].split()] - return boxes, dds, stds_ale, stds_epi, kk_list, dds_geom, xyzs, xy_kps + return boxes, dds, stds_ale, stds_epi, dds_geom except FileNotFoundError: - return [], [], [], [], [], [], [], [] + return [], [], [], [], [] - def _estimate_error_base(self, boxes, dds, boxes_gt, dds_gt, truncs_gt, occs_gt, method): + def _estimate_error(self, out_gt, out, method): + """Estimate localization error""" - # Compute error (distance) and save it - boxes_gt = copy.deepcopy(boxes_gt) - dds_gt = copy.deepcopy(dds_gt) - truncs_gt = copy.deepcopy(truncs_gt) - occs_gt = copy.deepcopy(occs_gt) + boxes_gt, _, dds_gt, truncs_gt, occs_gt = out_gt + if method == 'our': + boxes, dds, stds_ale, stds_epi, dds_geom = out + else: + boxes, dds = out - for idx, box in enumerate(boxes): - if len(boxes_gt) >= 1: - dd = dds[idx] - idx_max, iou_max = get_idx_max(box, boxes_gt) - cat = get_category(boxes_gt[idx_max], truncs_gt[idx_max], occs_gt[idx_max]) - # Update error if match is found - if iou_max > self.dic_thresh_iou[method]: - dd_gt = dds_gt[idx_max] - self.update_errors(dd, dd_gt, cat, self.errors[method]) + matches = get_iou_matches(boxes, boxes_gt, self.dic_thresh_iou[method]) - boxes_gt.pop(idx_max) - dds_gt.pop(idx_max) - truncs_gt.pop(idx_max) - occs_gt.pop(idx_max) - else: - break + for (idx, idx_gt) in matches: + # Update error if match is found + cat = get_category(boxes_gt[idx_gt], truncs_gt[idx_gt], occs_gt[idx_gt]) + self.update_errors(dds[idx], dds_gt[idx_gt], cat, self.errors[method]) + if method == 'our': + self.update_errors(dds_geom[idx], dds_gt[idx_gt], cat, self.errors['geom']) + self.update_uncertainty(stds_ale[idx], stds_epi[idx], dds[idx], dds_gt[idx_gt], cat) - def _estimate_error_mloco(self, boxes, dds, stds_ale, stds_epi, dds_geom, boxes_gt, dds_gt, truncs_gt, occs_gt): + def _compare_error(self, out_gt, out_m3d, out_3dop, out_md, out_our): + """Compare the error for a pool of instances commonly matched by all methods""" - # Compute error (distance) and save it - boxes_gt = copy.deepcopy(boxes_gt) - dds_gt = copy.deepcopy(dds_gt) - truncs_gt = copy.deepcopy(truncs_gt) - occs_gt = copy.deepcopy(occs_gt) + # Extract outputs of each method + boxes_gt, _, dds_gt, truncs_gt, occs_gt = out_gt + boxes_m3d, dds_m3d = out_m3d + boxes_3dop, dds_3dop = out_3dop + boxes_md, dds_md = out_md + boxes_our, dds_our, _, _, dds_geom = out_our - for idx, box in enumerate(boxes): - if len(boxes_gt) >= 1: - dd = dds[idx] - dd_geom = dds_geom[idx] - ale = stds_ale[idx] - epi = stds_epi[idx] - idx_max, iou_max = get_idx_max(box, boxes_gt) - cat = get_category(boxes_gt[idx_max], truncs_gt[idx_max], occs_gt[idx_max]) + # Find IoU matches + matches_our = get_iou_matches(boxes_our, boxes_gt, self.dic_thresh_iou['our']) + matches_m3d = get_iou_matches(boxes_m3d, boxes_gt, self.dic_thresh_iou['m3d']) + matches_3dop = get_iou_matches(boxes_3dop, boxes_gt, self.dic_thresh_iou['3dop']) + matches_md = get_iou_matches(boxes_md, boxes_gt, self.dic_thresh_iou['md']) - # Update error if match is found - if iou_max > self.dic_thresh_iou['our']: - dd_gt = dds_gt[idx_max] - self.update_errors(dd, dd_gt, cat, self.errors['our']) - self.update_errors(dd_geom, dd_gt, cat, self.errors['geom']) - self.update_uncertainty(ale, epi, dd, dd_gt, cat) - - boxes_gt.pop(idx_max) - dds_gt.pop(idx_max) - truncs_gt.pop(idx_max) - occs_gt.pop(idx_max) - - def _compare_error(self, boxes_m3d, dds_m3d, boxes_3dop, dds_3dop, boxes_md, dds_md, boxes_our, dds_our, - boxes_gt, dds_gt, truncs_gt, occs_gt, dds_geom): - - boxes_gt = copy.deepcopy(boxes_gt) - dds_gt = copy.deepcopy(dds_gt) - truncs_gt = copy.deepcopy(truncs_gt) - occs_gt = copy.deepcopy(occs_gt) - - for idx, box in enumerate(boxes_our): - if len(boxes_gt) >= 1: - dd_our = dds_our[idx] - dd_geom = dds_geom[idx] - idx_max, iou_max = get_idx_max(box, boxes_gt) - cat = get_category(boxes_gt[idx_max], truncs_gt[idx_max], occs_gt[idx_max]) - - idx_max_3dop, iou_max_3dop = get_idx_max(box, boxes_3dop) - idx_max_m3d, iou_max_m3d = get_idx_max(box, boxes_m3d) - idx_max_md, iou_max_md = get_idx_max(box, boxes_md) - - iou_min = min(iou_max_3dop, iou_max_m3d, iou_max_md) - - if iou_max >= self.dic_thresh_iou['our'] and iou_min >= self.dic_thresh_iou['m3d']: - dd_gt = dds_gt[idx_max] - dd_3dop = dds_3dop[idx_max_3dop] - dd_m3d = dds_m3d[idx_max_m3d] - dd_md = dds_md[idx_max_md] - - self.update_errors(dd_3dop, dd_gt, cat, self.errors['3dop_merged']) - self.update_errors(dd_our, dd_gt, cat, self.errors['our_merged']) - self.update_errors(dd_m3d, dd_gt, cat, self.errors['m3d_merged']) - self.update_errors(dd_geom, dd_gt, cat, self.errors['geom_merged']) - self.update_errors(dd_md, dd_gt, cat, self.errors['md_merged']) - self.dic_cnt['merged'] += 1 - - boxes_gt.pop(idx_max) - dds_gt.pop(idx_max) - truncs_gt.pop(idx_max) - occs_gt.pop(idx_max) - else: - break + # Update error of commonly matched instances + for (idx, idx_gt) in matches_our: + check, indices = extract_indices(idx_gt, matches_m3d, matches_3dop, matches_md) + if check: + cat = get_category(boxes_gt[idx_gt], truncs_gt[idx_gt], occs_gt[idx_gt]) + dd_gt = dds_gt[idx_gt] + self.update_errors(dds_our[idx], dd_gt, cat, self.errors['our_merged']) + self.update_errors(dds_geom[idx], dd_gt, cat, self.errors['geom_merged']) + self.update_errors(dds_m3d[indices[0]], dd_gt, cat, self.errors['m3d_merged']) + self.update_errors(dds_3dop[indices[1]], dd_gt, cat, self.errors['3dop_merged']) + self.update_errors(dds_md[indices[2]], dd_gt, cat, self.errors['md_merged']) + self.dic_cnt['merged'] += 1 def update_errors(self, dd, dd_gt, cat, errors): @@ -398,3 +329,25 @@ def find_cluster(dd, clusters): return clst return clusters[-1] + + +def extract_indices(idx_to_check, *args): + """ + Look if a given index j_gt is present in all the other series of indices (_, j) + and return the corresponding one for argument + + idx_check --> gt index to check for correspondences in other method + idx_method --> index corresponding to the method + idx_gt --> index gt of the method + idx_pred --> index of the predicted box of the method + indices --> list of predicted indices for each method corresponding to the ground truth index to check + """ + + checks = [False]*len(args) + indices = [] + for idx_method, method in enumerate(args): + for (idx_pred, idx_gt) in method: + if idx_gt == idx_to_check: + checks[idx_method] = True + indices.append(idx_pred) + return all(checks), indices diff --git a/src/features/preprocess_ki.py b/src/features/preprocess_ki.py index 18eb932..a4afec0 100644 --- a/src/features/preprocess_ki.py +++ b/src/features/preprocess_ki.py @@ -18,11 +18,11 @@ class PreprocessKitti: logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) - dic_jo = {'train': dict(X=[], Y=[], names=[], kps=[], K=[], + dic_jo = {'train': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[], clst=defaultdict(lambda: defaultdict(list))), - 'val': dict(X=[], Y=[], names=[], kps=[], K=[], + 'val': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[], clst=defaultdict(lambda: defaultdict(list))), - 'test': dict(X=[], Y=[], names=[], kps=[], K=[], + 'test': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[], clst=defaultdict(lambda: defaultdict(list)))} dic_names = defaultdict(lambda: defaultdict(list)) @@ -50,6 +50,8 @@ class PreprocessKitti: """Save json files""" cnt_gt = 0 + cnt_files = 0 + cnt_files_ped = 0 cnt_fnf = 0 dic_cnt = {'train': 0, 'val': 0, 'test': 0} @@ -68,11 +70,13 @@ class PreprocessKitti: kk = p_left[0] # Iterate over each line of the gt file and save box location and distances - boxes_gt, dds_gt, _, _ = parse_ground_truth(path_gt) + (boxes_gt, boxes_3d, dds_gt, _, _) = parse_ground_truth(path_gt) self.dic_names[basename + '.png']['boxes'] = copy.deepcopy(boxes_gt) self.dic_names[basename + '.png']['dds'] = copy.deepcopy(dds_gt) self.dic_names[basename + '.png']['K'] = copy.deepcopy(kk.tolist()) cnt_gt += len(boxes_gt) + cnt_files += 1 + cnt_files_ped += min(len(boxes_gt), 1) # if no boxes 0 else 1 # Find the annotations if exists try: @@ -93,7 +97,8 @@ class PreprocessKitti: self.dic_jo[phase]['kps'].append(uv_kps[ii]) self.dic_jo[phase]['X'].append(inputs[ii]) self.dic_jo[phase]['Y'].append([dds_gt[idx_max]]) # Trick to make it (nn,1) - self.dic_jo[phase]['K'] = kk.tolist() + self.dic_jo[phase]['boxes_3d'].append(boxes_3d[idx_max]) + self.dic_jo[phase]['K'].append(kk.tolist()) self.dic_jo[phase]['names'].append(name) # One image name for each annotation append_cluster(self.dic_jo, phase, inputs[ii], dds_gt[idx_max], uv_kps[ii]) dic_cnt[phase] += 1 @@ -107,8 +112,9 @@ class PreprocessKitti: for phase in ['train', 'val', 'test']: print("Saved {} annotations for phase {}" .format(dic_cnt[phase], phase)) - print("Number of GT files: {}. Files not found: {}" - .format(cnt_gt, cnt_fnf)) + print("Number of GT files: {}. Files with at least one pedestrian: {}. Files not found: {}" + .format(cnt_files, cnt_files_ped, cnt_fnf)) + print("Number of GT annotations: {}".format(cnt_gt)) print("\nOutput files:\n{}\n{}\n".format(self.path_names, self.path_joints)) def _factory_phase(self, name): diff --git a/src/features/preprocess_nu.py b/src/features/preprocess_nu.py index ac0790c..e1a8478 100644 --- a/src/features/preprocess_nu.py +++ b/src/features/preprocess_nu.py @@ -133,7 +133,7 @@ class PreprocessNuscenes: self.dic_jo[phase]['Y'].append([dds[idx_max]]) # Trick to make it (nn,1) self.dic_jo[phase]['names'].append(name) # One image name for each annotation self.dic_jo[phase]['boxes_3d'].append(boxes_3d[idx_max]) - self.dic_jo[phase]['K'] = kk.tolist() + self.dic_jo[phase]['K'].append(kk.tolist()) append_cluster(self.dic_jo, phase, inputs[ii], dds[idx_max], uv_kps[ii]) boxes_gt.pop(idx_max) dds.pop(idx_max) diff --git a/src/utils/camera.py b/src/utils/camera.py index 9b1824e..24c2891 100644 --- a/src/utils/camera.py +++ b/src/utils/camera.py @@ -75,9 +75,6 @@ def preprocess_single(kps, kk): uv_kp = np.array([kps[0][idx], kps[1][idx], 1]) kps_uv.append(uv_kp) - # Take the ground joint - vv_gr = max(kps[1]) - # Projection in normalized image coordinates and zero-center with the center of the bounding box xy1_center = pixel_to_camera(uv_center, kk, 1) * 10 for idx, kp in enumerate(kps_uv): diff --git a/src/utils/kitti.py b/src/utils/kitti.py index 426a9ea..7f9304c 100644 --- a/src/utils/kitti.py +++ b/src/utils/kitti.py @@ -129,10 +129,12 @@ def get_category(box, trunc, occ): if hh >= 40 and trunc <= 0.15 and occ <= 0: cat = 'easy' - elif trunc <= 0.3 and occ <= 1: + elif trunc <= 0.3 and occ <= 1 and hh >= 25: cat = 'moderate' - else: + elif trunc <= 0.5 and occ <= 2 and hh >= 25: cat = 'hard' + else: + cat = 'excluded' return cat @@ -162,6 +164,7 @@ def parse_ground_truth(path_gt): dds_gt = [] truncs_gt = [] # Float from 0 to 1 occs_gt = [] # Either 0,1,2,3 fully visible, partly occluded, largely occluded, unknown + boxes_3d = [] with open(path_gt, "r") as f_gt: for line_gt in f_gt: @@ -170,6 +173,8 @@ def parse_ground_truth(path_gt): occs_gt.append(int(line_gt.split()[2])) boxes_gt.append([float(x) for x in line_gt.split()[4:8]]) loc_gt = [float(x) for x in line_gt.split()[11:14]] + wlh = [float(x) for x in line_gt.split()[8:11]] + boxes_3d.append(loc_gt + wlh) dds_gt.append(math.sqrt(loc_gt[0] ** 2 + loc_gt[1] ** 2 + loc_gt[2] ** 2)) - return boxes_gt, dds_gt, truncs_gt, occs_gt \ No newline at end of file + return (boxes_gt, boxes_3d, dds_gt, truncs_gt, occs_gt) diff --git a/src/utils/misc.py b/src/utils/misc.py index 30e411c..aa1a3fa 100644 --- a/src/utils/misc.py +++ b/src/utils/misc.py @@ -64,6 +64,38 @@ def get_idx_max(box, boxes_gt): return idx_max, iou_max +def get_iou_matrix(boxes, boxes_gt): + """ + Get IoU matrix between predicted and ground truth boxes + Dim: (boxes, boxes_gt) + """ + iou_matrix = np.zeros((len(boxes), len(boxes_gt))) + + for idx, box in enumerate(boxes): + for idx_gt, box_gt in enumerate(boxes_gt): + iou_matrix[idx, idx_gt] = calculate_iou(box, box_gt) + return iou_matrix + + +def get_iou_matches(boxes, boxes_gt, thresh): + """From 2 sets of boxes and a minimum threshold, compute the matching indices for IoU matchings""" + + iou_matrix = get_iou_matrix(boxes, boxes_gt) + if not iou_matrix.size: + return [] + + matches = [] + iou_max = np.max(iou_matrix) + while iou_max > thresh: + # Extract the indeces of the max + args_max = np.unravel_index(np.argmax(iou_matrix, axis=None), iou_matrix.shape) + matches.append(args_max) + iou_matrix[args_max[0], :] = 0 + iou_matrix[:, args_max[1]] = 0 + iou_max = np.max(iou_matrix) + return matches + + def reparametrize_box3d(box): """Reparametrized 3D box in the XZ plane and add the height""" diff --git a/src/utils/stereo.py b/src/utils/stereo.py deleted file mode 100644 index e744017..0000000 --- a/src/utils/stereo.py +++ /dev/null @@ -1,49 +0,0 @@ - -import copy -import numpy as np - - -def depth_from_disparity(zzs, zzs_right, kps, kps_right): - """Associate instances in left and right images and compute disparity""" - - zzs_stereo = [] - cnt = 0 - for idx, zz in enumerate(zzs): - - # Find the closest human in terms of distance - zz_stereo, idx_min, delta_d_min = calculate_disparity(zz, zzs_right, kps[idx], kps_right) - if delta_d_min < 1: - zzs_stereo.append(zz_stereo) - zzs_right.pop(idx_min) - kps_right.pop(idx_min) - cnt += 1 - else: - zzs_stereo.append(zz) - - return zzs_stereo, cnt - - -def calculate_disparity(zz, zzs_right, kp, kps_right): - """From 2 sets of keypoints calculate disparity as the median of the disparities""" - - kp = np.array(copy.deepcopy(kp)) - kps_right = np.array(copy.deepcopy(kps_right)) - zz_stereo = 0 - idx_min = 0 - delta_z_min = 4 - - for idx, zz_right in enumerate(zzs_right): - delta_z = abs(zz - zz_right) - diffs = np.array(np.array(kp[0] - kps_right[idx][0])) - diff = np.mean(diffs) - - # Check only for right instances (5 pxls = 80meters) - if delta_z < delta_z_min and diff > 5: - delta_z_min = delta_z - idx_min = idx - zzs = 0.54 * 721 / diffs - zz_stereo = np.median(zzs[kp[2] > 0]) - - return zz_stereo, idx_min, delta_z_min - - diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..c9c11dd --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,11 @@ + + +from utils.misc import get_iou_matrix + + +def test_iou(): + boxes_pred = [[1, 100, 1, 200]] + boxes_gt = [[100., 120., 150., 160.],[12, 110, 130., 160.]] + iou_matrix = get_iou_matrix(boxes_pred, boxes_gt) + assert iou_matrix.shape == (len(boxes_pred), len(boxes_gt)) +