239 lines
9.1 KiB
Python
239 lines
9.1 KiB
Python
|
|
# pylint: disable=too-many-statements,cyclic-import, too-many-branches
|
|
|
|
"""Joints Analysis: Supplementary material of MonStereo"""
|
|
|
|
import json
|
|
import os
|
|
from collections import defaultdict
|
|
|
|
import numpy as np
|
|
import matplotlib.pyplot as plt
|
|
|
|
from .eval_kitti import find_cluster, average
|
|
from ..visuals.figures import get_distances
|
|
|
|
COCO_KEYPOINTS = [
|
|
'nose', # 0
|
|
'left_eye', # 1
|
|
'right_eye', # 2
|
|
'left_ear', # 3
|
|
'right_ear', # 4
|
|
'left_shoulder', # 5
|
|
'right_shoulder', # 6
|
|
'left_elbow', # 7
|
|
'right_elbow', # 8
|
|
'left_wrist', # 9
|
|
'right_wrist', # 10
|
|
'left_hip', # 11
|
|
'right_hip', # 12
|
|
'left_knee', # 13
|
|
'right_knee', # 14
|
|
'left_ankle', # 15
|
|
'right_ankle', # 16
|
|
]
|
|
|
|
|
|
def joints_variance(joints, clusters, dic_ms):
|
|
# CLUSTERS = ('3', '5', '7', '9', '11', '13', '15', '17', '19', '21', '23', '25', '27', '29', '31', '49')
|
|
BF = 0.54 * 721
|
|
phase = 'train'
|
|
methods = ('pifpaf', 'mask')
|
|
dic_fin = {}
|
|
|
|
for method in methods:
|
|
dic_var = defaultdict(lambda: defaultdict(list))
|
|
dic_joints = defaultdict(list)
|
|
dic_avg = defaultdict(lambda: defaultdict(float))
|
|
path_joints = joints + '_' + method + '.json'
|
|
|
|
with open(path_joints, 'r') as f:
|
|
dic_jo = json.load(f)
|
|
|
|
for idx, keypoint in enumerate(dic_jo[phase]['kps']):
|
|
# if dic_jo[phase]['names'][idx] == '005856.txt' and dic_jo[phase]['Y'][idx][2] > 14:
|
|
# aa = 4
|
|
assert len(keypoint) < 2
|
|
kps = np.array(keypoint[0])[:, :17]
|
|
kps_r = np.array(keypoint[0])[:, 17:]
|
|
disps = kps[0] - kps_r[0]
|
|
zz = dic_jo[phase]['Y'][idx][2]
|
|
disps_3 = get_variance(kps, kps_r, zz)
|
|
disps_8 = get_variance_conf(kps, kps_r, num=8)
|
|
disps_4 = get_variance_conf(kps, kps_r, num=4)
|
|
disp_gt = BF / zz
|
|
clst = find_cluster(zz, clusters) # 4 = '3' 35 = '31' 42 = 2 = 'excl'
|
|
dic_var['std_d'][clst].append(disps.std())
|
|
errors = np.minimum(30, np.abs(zz - BF / disps))
|
|
dic_var['mean_dev'][clst].append(min(30, abs(zz - BF / np.median(disps))))
|
|
dic_var['mean_3'][clst].append(min(30, abs(zz - BF / disps_3.mean())))
|
|
dic_var['mean_8'][clst].append(min(30, abs(zz - BF / np.median(disps_8))))
|
|
dic_var['mean_4'][clst].append(min(30, abs(zz - BF / np.median(disps_4))))
|
|
arg_best = np.argmin(errors)
|
|
conf = np.mean((kps[2][arg_best], kps_r[2][arg_best]))
|
|
dic_var['mean_best'][clst].append(np.min(errors))
|
|
dic_var['conf_best'][clst].append(conf)
|
|
dic_var['conf'][clst].append(np.mean((np.mean(kps[2]), np.mean(kps_r[2]))))
|
|
# dic_var['std_z'][clst].append(zzs.std())
|
|
for ii, el in enumerate(disps):
|
|
if abs(el-disp_gt) < 1:
|
|
dic_var['rep'][clst].append(1)
|
|
dic_joints[str(ii)].append(1)
|
|
else:
|
|
dic_var['rep'][clst].append(0)
|
|
dic_joints[str(ii)].append(0)
|
|
|
|
for key in dic_var:
|
|
for clst in clusters[:-1]: # 41 needs to be excluded (36 = '31')
|
|
dic_avg[key][clst] = average(dic_var[key][clst])
|
|
dic_fin[method] = dic_avg
|
|
for key in dic_joints:
|
|
dic_fin[method]['joints'][key] = average(dic_joints[key])
|
|
dic_fin['monstereo'] = {clst: dic_ms[clst]['mean'] for clst in clusters[:-1]}
|
|
variance_figures(dic_fin, clusters)
|
|
|
|
|
|
def get_variance(kps, kps_r, zz):
|
|
|
|
thresh = 0.5 - zz / 100
|
|
disps_2 = []
|
|
disps = kps[0] - kps_r[0]
|
|
arg_disp = np.argsort(disps)[::-1]
|
|
|
|
for idx in arg_disp[1:]:
|
|
if kps[2][idx] > thresh and kps_r[2][idx] > thresh:
|
|
disps_2.append(disps[idx])
|
|
if len(disps_2) >= 3:
|
|
return np.array(disps_2)
|
|
return disps
|
|
|
|
|
|
def get_variance_conf(kps, kps_r, num=8):
|
|
|
|
disps_conf = []
|
|
confs = (kps[2, :] + kps_r[2, :]) / 2
|
|
disps = kps[0] - kps_r[0]
|
|
arg_disp = np.argsort(confs)[::-1]
|
|
|
|
for idx in arg_disp[:num]:
|
|
disps_conf.append(disps[idx])
|
|
return np.array(disps_conf)
|
|
|
|
|
|
def variance_figures(dic_fin, clusters):
|
|
"""Predicted confidence intervals and task error as a function of ground-truth distance"""
|
|
|
|
dir_out = 'docs'
|
|
x_min = 3
|
|
x_max = 43
|
|
y_min = 0
|
|
y_max = 1
|
|
|
|
plt.figure(0)
|
|
plt.xlabel("Ground-truth distance [m]")
|
|
plt.title("Repeatability by distance")
|
|
plt.xlim(x_min, x_max)
|
|
plt.ylim(y_min, y_max)
|
|
plt.grid(linewidth=0.2)
|
|
|
|
xxs = get_distances(clusters)
|
|
yys_p = [el for _, el in dic_fin['pifpaf']['rep'].items()]
|
|
yys_m = [el for _, el in dic_fin['mask']['rep'].items()]
|
|
plt.plot(xxs, yys_p, marker='s', label="PifPaf")
|
|
plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN")
|
|
plt.tight_layout()
|
|
plt.legend()
|
|
path_fig = os.path.join(dir_out, 'repeatability.png')
|
|
plt.savefig(path_fig)
|
|
print("Figure of repeatability saved in {}".format(path_fig))
|
|
|
|
plt.figure(1)
|
|
plt.xlabel("Ground-truth distance [m]")
|
|
plt.ylabel("[m]")
|
|
plt.title("Depth error")
|
|
plt.grid(linewidth=0.2)
|
|
y_min = 0
|
|
y_max = 2.7
|
|
plt.ylim(y_min, y_max)
|
|
yys_p = [el for _, el in dic_fin['pifpaf']['mean_dev'].items()]
|
|
# yys_m = [el for _, el in dic_fin['mask']['mean_dev'].items()]
|
|
yys_p_3 = [el for _, el in dic_fin['pifpaf']['mean_3'].items()]
|
|
yys_p_8 = [el for _, el in dic_fin['pifpaf']['mean_8'].items()]
|
|
yys_p_4 = [el for _, el in dic_fin['pifpaf']['mean_4'].items()]
|
|
# yys_m_3 = [el for _, el in dic_fin['mask']['mean_3'].items()]
|
|
yys_ms = [el for _, el in dic_fin['monstereo'].items()]
|
|
yys_p_best = [el for _, el in dic_fin['pifpaf']['mean_best'].items()]
|
|
plt.plot(xxs, yys_p_4, marker='o', linestyle=':', label="PifPaf (highest 4)")
|
|
plt.plot(xxs, yys_p, marker='+', label="PifPaf (median)")
|
|
# plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN (median")
|
|
plt.plot(xxs, yys_p_3, marker='s', linestyle='--', label="PifPaf (closest 3)")
|
|
plt.plot(xxs, yys_p_8, marker='*', linestyle=':', label="PifPaf (highest 8)")
|
|
plt.plot(xxs, yys_ms, marker='^', label="MonStereo")
|
|
plt.plot(xxs, yys_p_best, marker='o', label="PifPaf (best)")
|
|
# plt.plot(xxs, yys_m_3, marker='o', color='r', label="Mask R-CNN (closest 3)")
|
|
# plt.plot(xxs, yys_mon, marker='o', color='b', label="Our MonStereo")
|
|
|
|
plt.legend()
|
|
plt.tight_layout()
|
|
path_fig = os.path.join(dir_out, 'mean_deviation.png')
|
|
plt.savefig(path_fig)
|
|
print("Figure of mean deviation saved in {}".format(path_fig))
|
|
|
|
plt.figure(2)
|
|
plt.xlabel("Ground-truth distance [m]")
|
|
plt.ylabel("Pixels")
|
|
plt.title("Standard deviation of joints disparity")
|
|
yys_p = [el for _, el in dic_fin['pifpaf']['std_d'].items()]
|
|
yys_m = [el for _, el in dic_fin['mask']['std_d'].items()]
|
|
yys_p_z = [el for _, el in dic_fin['pifpaf']['std_z'].items()]
|
|
yys_m_z = [el for _, el in dic_fin['mask']['std_z'].items()]
|
|
plt.plot(xxs, yys_p, marker='s', label="PifPaf")
|
|
plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN")
|
|
# plt.plot(xxs, yys_p_z, marker='s', color='b', label="PifPaf (meters)")
|
|
# plt.plot(xxs, yys_m_z, marker='o', color='r', label="Mask R-CNN (meters)")
|
|
|
|
plt.grid(linewidth=0.2)
|
|
plt.legend()
|
|
path_fig = os.path.join(dir_out, 'std_joints.png')
|
|
plt.savefig(path_fig)
|
|
print("Figure of standard deviation of joints by distance in {}".format(path_fig))
|
|
|
|
plt.figure(3)
|
|
# plt.style.use('ggplot')
|
|
width = 0.35
|
|
xxs = np.arange(len(COCO_KEYPOINTS))
|
|
yys_p = [el for _, el in dic_fin['pifpaf']['joints'].items()]
|
|
yys_m = [el for _, el in dic_fin['mask']['joints'].items()]
|
|
plt.bar(xxs, yys_p, width, color='C0', label='Pifpaf')
|
|
plt.bar(xxs + width, yys_m, width, color='C1', label='Mask R-CNN')
|
|
plt.ylim(0, 1)
|
|
|
|
plt.xlabel("Keypoints")
|
|
plt.title("Repeatability by keypoint type")
|
|
|
|
plt.xticks(xxs + width / 2, xxs)
|
|
plt.legend(loc='best')
|
|
path_fig = os.path.join(dir_out, 'repeatability_2.png')
|
|
plt.savefig(path_fig)
|
|
plt.close('all')
|
|
print("Figure of standard deviation of joints by keypointd in {}".format(path_fig))
|
|
|
|
plt.figure(4)
|
|
plt.xlabel("Ground-truth distance [m]")
|
|
plt.ylabel("Confidence")
|
|
plt.grid(linewidth=0.2)
|
|
xxs = get_distances(clusters)
|
|
yys_p_conf = [el for _, el in dic_fin['pifpaf']['conf'].items()]
|
|
yys_p_conf_best = [el for _, el in dic_fin['pifpaf']['conf_best'].items()]
|
|
yys_m_conf = [el for _, el in dic_fin['mask']['conf'].items()]
|
|
yys_m_conf_best = [el for _, el in dic_fin['mask']['conf_best'].items()]
|
|
plt.plot(xxs, yys_p_conf_best, marker='s', color='lightblue', label="PifPaf (best)")
|
|
plt.plot(xxs, yys_p_conf, marker='s', color='b', label="PifPaf (mean)")
|
|
plt.plot(xxs, yys_m_conf_best, marker='^', color='darkorange', label="Mask (best)")
|
|
plt.plot(xxs, yys_m_conf, marker='o', color='r', label="Mask R-CNN (mean)")
|
|
plt.legend()
|
|
plt.tight_layout()
|
|
path_fig = os.path.join(dir_out, 'confidence.png')
|
|
plt.savefig(path_fig)
|
|
print("Figure of confidence saved in {}".format(path_fig))
|