import os
from time import strftime, localtime
import numpy as np
from utils import get_msg_mgr, mkdir
from .metric import mean_iou, cuda_dist, compute_ACC_mAP, evaluate_rank, evaluate_many
from .re_rank import re_ranking
from sklearn.metrics import confusion_matrix, accuracy_score
def de_diag(acc, each_angle=False):
# Exclude identical-view cases
dividend = acc.shape[1] - 1.
result = np.sum(acc - np.diag(np.diag(acc)), 1) / dividend
if not each_angle:
result = np.mean(result)
return result
def cross_view_gallery_evaluation(feature, label, seq_type, view, dataset, metric):
'''More details can be found: More details can be found in
[A Comprehensive Study on the Evaluation of Silhouette-based Gait Recognition](https://ieeexplore.ieee.org/document/9928336).
'''
probe_seq_dict = {'CASIA-B': {'NM': ['nm-01'], 'BG': ['bg-01'], 'CL': ['cl-01']},
'OUMVLP': {'NM': ['00']}}
gallery_seq_dict = {'CASIA-B': ['nm-02', 'bg-02', 'cl-02'],
'OUMVLP': ['01']}
msg_mgr = get_msg_mgr()
acc = {}
mean_ap = {}
view_list = sorted(np.unique(view))
for (type_, probe_seq) in probe_seq_dict[dataset].items():
acc[type_] = np.zeros(len(view_list)) - 1.
mean_ap[type_] = np.zeros(len(view_list)) - 1.
for (v1, probe_view) in enumerate(view_list):
pseq_mask = np.isin(seq_type, probe_seq) & np.isin(
view, probe_view)
probe_x = feature[pseq_mask, :]
probe_y = label[pseq_mask]
gseq_mask = np.isin(seq_type, gallery_seq_dict[dataset])
gallery_y = label[gseq_mask]
gallery_x = feature[gseq_mask, :]
dist = cuda_dist(probe_x, gallery_x, metric)
eval_results = compute_ACC_mAP(
dist.cpu().numpy(), probe_y, gallery_y, view[pseq_mask], view[gseq_mask])
acc[type_][v1] = np.round(eval_results[0] * 100, 2)
mean_ap[type_][v1] = np.round(eval_results[1] * 100, 2)
result_dict = {}
msg_mgr.log_info(
'===Cross View Gallery Evaluation (Excluded identical-view cases)===')
out_acc_str = "========= Rank@1 Acc =========\n"
out_map_str = "============= mAP ============\n"
for type_ in probe_seq_dict[dataset].keys():
avg_acc = np.mean(acc[type_])
avg_map = np.mean(mean_ap[type_])
result_dict[f'scalar/test_accuracy/{type_}-Rank@1'] = avg_acc
result_dict[f'scalar/test_accuracy/{type_}-mAP'] = avg_map
out_acc_str += f"{type_}:\t{acc[type_]}, mean: {avg_acc:.2f}%\n"
out_map_str += f"{type_}:\t{mean_ap[type_]}, mean: {avg_map:.2f}%\n"
# msg_mgr.log_info(f'========= Rank@1 Acc =========')
msg_mgr.log_info(f'{out_acc_str}')
# msg_mgr.log_info(f'========= mAP =========')
msg_mgr.log_info(f'{out_map_str}')
return result_dict
# Modified From https://github.com/AbnerHqC/GaitSet/blob/master/model/utils/evaluator.py
def single_view_gallery_evaluation(feature, label, seq_type, view, dataset, metric):
probe_seq_dict = {'CASIA-B': {'NM': ['nm-05', 'nm-06'], 'BG': ['bg-01', 'bg-02'], 'CL': ['cl-01', 'cl-02']},
'OUMVLP': {'NM': ['00']},
'CASIA-E': {'NM': ['H-scene2-nm-1', 'H-scene2-nm-2', 'L-scene2-nm-1', 'L-scene2-nm-2', 'H-scene3-nm-1', 'H-scene3-nm-2', 'L-scene3-nm-1', 'L-scene3-nm-2', 'H-scene3_s-nm-1', 'H-scene3_s-nm-2', 'L-scene3_s-nm-1', 'L-scene3_s-nm-2', ],
'BG': ['H-scene2-bg-1', 'H-scene2-bg-2', 'L-scene2-bg-1', 'L-scene2-bg-2', 'H-scene3-bg-1', 'H-scene3-bg-2', 'L-scene3-bg-1', 'L-scene3-bg-2', 'H-scene3_s-bg-1', 'H-scene3_s-bg-2', 'L-scene3_s-bg-1', 'L-scene3_s-bg-2'],
'CL': ['H-scene2-cl-1', 'H-scene2-cl-2', 'L-scene2-cl-1', 'L-scene2-cl-2', 'H-scene3-cl-1', 'H-scene3-cl-2', 'L-scene3-cl-1', 'L-scene3-cl-2', 'H-scene3_s-cl-1', 'H-scene3_s-cl-2', 'L-scene3_s-cl-1', 'L-scene3_s-cl-2']
},
'SUSTech1K': {'Normal': ['01-nm'], 'Bag': ['bg'], 'Clothing': ['cl'], 'Carrying':['cr'], 'Umberalla': ['ub'], 'Uniform': ['uf'], 'Occlusion': ['oc'],'Night': ['nt'], 'Overall': ['01','02','03','04']}
}
gallery_seq_dict = {'CASIA-B': ['nm-01', 'nm-02', 'nm-03', 'nm-04'],
'OUMVLP': ['01'],
'CASIA-E': ['H-scene1-nm-1', 'H-scene1-nm-2', 'L-scene1-nm-1', 'L-scene1-nm-2'],
'SUSTech1K': ['00-nm'],}
msg_mgr = get_msg_mgr()
acc = {}
view_list = sorted(np.unique(view))
num_rank = 1
if dataset == 'CASIA-E':
view_list.remove("270")
if dataset == 'SUSTech1K':
num_rank = 5
view_num = len(view_list)
for (type_, probe_seq) in probe_seq_dict[dataset].items():
acc[type_] = np.zeros((view_num, view_num, num_rank)) - 1.
for (v1, probe_view) in enumerate(view_list):
pseq_mask = np.isin(seq_type, probe_seq) & np.isin(
view, probe_view)
pseq_mask = pseq_mask if 'SUSTech1K' not in dataset else np.any(np.asarray(
[np.char.find(seq_type, probe)>=0 for probe in probe_seq]), axis=0
) & np.isin(view, probe_view) # For SUSTech1K only
probe_x = feature[pseq_mask, :]
probe_y = label[pseq_mask]
for (v2, gallery_view) in enumerate(view_list):
gseq_mask = np.isin(seq_type, gallery_seq_dict[dataset]) & np.isin(
view, [gallery_view])
gseq_mask = gseq_mask if 'SUSTech1K' not in dataset else np.any(np.asarray(
[np.char.find(seq_type, gallery)>=0 for gallery in gallery_seq_dict[dataset]]), axis=0
) & np.isin(view, [gallery_view]) # For SUSTech1K only
gallery_y = label[gseq_mask]
gallery_x = feature[gseq_mask, :]
dist = cuda_dist(probe_x, gallery_x, metric)
idx = dist.topk(num_rank, largest=False)[1].cpu().numpy()
acc[type_][v1, v2, :] = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0,
0) * 100 / dist.shape[0], 2)
result_dict = {}
msg_mgr.log_info('===Rank-1 (Exclude identical-view cases)===')
out_str = ""
for rank in range(num_rank):
out_str = ""
for type_ in probe_seq_dict[dataset].keys():
sub_acc = de_diag(acc[type_][:,:,rank], each_angle=True)
if rank == 0:
msg_mgr.log_info(f'{type_}@R{rank+1}: {sub_acc}')
result_dict[f'scalar/test_accuracy/{type_}@R{rank+1}'] = np.mean(sub_acc)
out_str += f"{type_}@R{rank+1}: {np.mean(sub_acc):.2f}%\t"
msg_mgr.log_info(out_str)
return result_dict
def evaluate_indoor_dataset(data, dataset, metric='euc', cross_view_gallery=False):
feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views']
label = np.array(label)
view = np.array(view)
if dataset not in ('CASIA-B', 'OUMVLP', 'CASIA-E', 'SUSTech1K'):
raise KeyError("DataSet %s hasn't been supported !" % dataset)
if cross_view_gallery:
return cross_view_gallery_evaluation(
feature, label, seq_type, view, dataset, metric)
else:
return single_view_gallery_evaluation(
feature, label, seq_type, view, dataset, metric)
def evaluate_real_scene(data, dataset, metric='euc'):
msg_mgr = get_msg_mgr()
feature, label, seq_type = data['embeddings'], data['labels'], data['types']
label = np.array(label)
gallery_seq_type = {'0001-1000': ['1', '2'],
"HID2021": ['0'], '0001-1000-test': ['0'],
'GREW': ['01'], 'TTG-200': ['1']}
probe_seq_type = {'0001-1000': ['3', '4', '5', '6'],
"HID2021": ['1'], '0001-1000-test': ['1'],
'GREW': ['02'], 'TTG-200': ['2', '3', '4', '5', '6']}
num_rank = 20
acc = np.zeros([num_rank]) - 1.
gseq_mask = np.isin(seq_type, gallery_seq_type[dataset])
gallery_x = feature[gseq_mask, :]
gallery_y = label[gseq_mask]
pseq_mask = np.isin(seq_type, probe_seq_type[dataset])
probe_x = feature[pseq_mask, :]
probe_y = label[pseq_mask]
dist = cuda_dist(probe_x, gallery_x, metric)
idx = dist.topk(num_rank, largest=False)[1].cpu().numpy()
acc = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0,
0) * 100 / dist.shape[0], 2)
msg_mgr.log_info('==Rank-1==')
msg_mgr.log_info('%.3f' % (np.mean(acc[0])))
msg_mgr.log_info('==Rank-5==')
msg_mgr.log_info('%.3f' % (np.mean(acc[4])))
msg_mgr.log_info('==Rank-10==')
msg_mgr.log_info('%.3f' % (np.mean(acc[9])))
msg_mgr.log_info('==Rank-20==')
msg_mgr.log_info('%.3f' % (np.mean(acc[19])))
return {"scalar/test_accuracy/Rank-1": np.mean(acc[0]), "scalar/test_accuracy/Rank-5": np.mean(acc[4])}
def GREW_submission(data, dataset, metric='euc'):
get_msg_mgr().log_info("Evaluating GREW")
feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views']
label = np.array(label)
view = np.array(view)
gallery_seq_type = {'GREW': ['01', '02']}
probe_seq_type = {'GREW': ['03']}
gseq_mask = np.isin(seq_type, gallery_seq_type[dataset])
gallery_x = feature[gseq_mask, :]
gallery_y = label[gseq_mask]
pseq_mask = np.isin(seq_type, probe_seq_type[dataset])
probe_x = feature[pseq_mask, :]
probe_y = view[pseq_mask]
num_rank = 20
dist = cuda_dist(probe_x, gallery_x, metric)
idx = dist.topk(num_rank, largest=False)[1].cpu().numpy()
save_path = os.path.join(
"GREW_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv")
mkdir("GREW_result")
with open(save_path, "w") as f:
f.write("videoId,rank1,rank2,rank3,rank4,rank5,rank6,rank7,rank8,rank9,rank10,rank11,rank12,rank13,rank14,rank15,rank16,rank17,rank18,rank19,rank20\n")
for i in range(len(idx)):
r_format = [int(idx) for idx in gallery_y[idx[i, 0:num_rank]]]
output_row = '{}'+',{}'*num_rank+'\n'
f.write(output_row.format(probe_y[i], *r_format))
print("GREW result saved to {}/{}".format(os.getcwd(), save_path))
return
def HID_submission(data, dataset, rerank=True, metric='euc'):
msg_mgr = get_msg_mgr()
msg_mgr.log_info("Evaluating HID")
feature, label, seq_type = data['embeddings'], data['labels'], data['views']
label = np.array(label)
seq_type = np.array(seq_type)
probe_mask = (label == "probe")
gallery_mask = (label != "probe")
gallery_x = feature[gallery_mask, :]
gallery_y = label[gallery_mask]
probe_x = feature[probe_mask, :]
probe_y = seq_type[probe_mask]
if rerank:
feat = np.concatenate([probe_x, gallery_x])
dist = cuda_dist(feat, feat, metric).cpu().numpy()
msg_mgr.log_info("Starting Re-ranking")
re_rank = re_ranking(
dist, probe_x.shape[0], k1=6, k2=6, lambda_value=0.3)
idx = np.argsort(re_rank, axis=1)
else:
dist = cuda_dist(probe_x, gallery_x, metric)
idx = dist.cpu().sort(1)[1].numpy()
save_path = os.path.join(
"HID_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv")
mkdir("HID_result")
with open(save_path, "w") as f:
f.write("videoID,label\n")
for i in range(len(idx)):
f.write("{},{}\n".format(probe_y[i], gallery_y[idx[i, 0]]))
print("HID result saved to {}/{}".format(os.getcwd(), save_path))
return
def evaluate_segmentation(data, dataset):
labels = data['mask']
pred = data['pred']
miou = mean_iou(pred, labels)
get_msg_mgr().log_info('mIOU: %.3f' % (miou.mean()))
return {"scalar/test_accuracy/mIOU": miou}
def evaluate_Gait3D(data, dataset, metric='euc'):
msg_mgr = get_msg_mgr()
features, labels, cams, time_seqs = data['embeddings'], data['labels'], data['types'], data['views']
import json
probe_sets = json.load(
open('./datasets/Gait3D/Gait3D.json', 'rb'))['PROBE_SET']
probe_mask = []
for id, ty, sq in zip(labels, cams, time_seqs):
if '-'.join([id, ty, sq]) in probe_sets:
probe_mask.append(True)
else:
probe_mask.append(False)
probe_mask = np.array(probe_mask)
# probe_features = features[:probe_num]
probe_features = features[probe_mask]
# gallery_features = features[probe_num:]
gallery_features = features[~probe_mask]
# probe_lbls = np.asarray(labels[:probe_num])
# gallery_lbls = np.asarray(labels[probe_num:])
probe_lbls = np.asarray(labels)[probe_mask]
gallery_lbls = np.asarray(labels)[~probe_mask]
results = {}
msg_mgr.log_info(f"The test metric you choose is {metric}.")
dist = cuda_dist(probe_features, gallery_features, metric).cpu().numpy()
cmc, all_AP, all_INP = evaluate_rank(dist, probe_lbls, gallery_lbls)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
results['scalar/test_accuracy/Rank-{}'.format(r)] = cmc[r - 1] * 100
results['scalar/test_accuracy/mAP'] = mAP * 100
results['scalar/test_accuracy/mINP'] = mINP * 100
# print_csv_format(dataset_name, results)
msg_mgr.log_info(results)
return results
def evaluate_CCPG(data, dataset, metric='euc'):
msg_mgr = get_msg_mgr()
feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views']
label = np.array(label)
for i in range(len(view)):
view[i] = view[i].split("_")[0]
view_np = np.array(view)
view_list = list(set(view))
view_list.sort()
view_num = len(view_list)
probe_seq_dict = {'CCPG': [["U0_D0_BG", "U0_D0"], [
"U3_D3"], ["U1_D0"], ["U0_D0_BG"]]}
gallery_seq_dict = {
'CCPG': [["U1_D1", "U2_D2", "U3_D3"], ["U0_D3"], ["U1_D1"], ["U0_D0"]]}
if dataset not in (probe_seq_dict or gallery_seq_dict):
raise KeyError("DataSet %s hasn't been supported !" % dataset)
num_rank = 5
acc = np.zeros([len(probe_seq_dict[dataset]),
view_num, view_num, num_rank]) - 1.
ap_save = []
cmc_save = []
minp = []
for (p, probe_seq) in enumerate(probe_seq_dict[dataset]):
# for gallery_seq in gallery_seq_dict[dataset]:
gallery_seq = gallery_seq_dict[dataset][p]
gseq_mask = np.isin(seq_type, gallery_seq)
gallery_x = feature[gseq_mask, :]
# print("gallery_x", gallery_x.shape)
gallery_y = label[gseq_mask]
gallery_view = view_np[gseq_mask]
pseq_mask = np.isin(seq_type, probe_seq)
probe_x = feature[pseq_mask, :]
probe_y = label[pseq_mask]
probe_view = view_np[pseq_mask]
msg_mgr.log_info(
("gallery length", len(gallery_y), gallery_seq, "probe length", len(probe_y), probe_seq))
distmat = cuda_dist(probe_x, gallery_x, metric).cpu().numpy()
# cmc, ap = evaluate(distmat, probe_y, gallery_y, probe_view, gallery_view)
cmc, ap, inp = evaluate_many(
distmat, probe_y, gallery_y, probe_view, gallery_view)
ap_save.append(ap)
cmc_save.append(cmc[0])
minp.append(inp)
# print(ap_save, cmc_save)
msg_mgr.log_info(
'===Rank-1 (Exclude identical-view cases for Person Re-Identification)===')
msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % (
cmc_save[0]*100, cmc_save[1]*100, cmc_save[2]*100, cmc_save[3]*100))
msg_mgr.log_info(
'===mAP (Exclude identical-view cases for Person Re-Identification)===')
msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % (
ap_save[0]*100, ap_save[1]*100, ap_save[2]*100, ap_save[3]*100))
msg_mgr.log_info(
'===mINP (Exclude identical-view cases for Person Re-Identification)===')
msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' %
(minp[0]*100, minp[1]*100, minp[2]*100, minp[3]*100))
for (p, probe_seq) in enumerate(probe_seq_dict[dataset]):
# for gallery_seq in gallery_seq_dict[dataset]:
gallery_seq = gallery_seq_dict[dataset][p]
for (v1, probe_view) in enumerate(view_list):
for (v2, gallery_view) in enumerate(view_list):
gseq_mask = np.isin(seq_type, gallery_seq) & np.isin(
view, [gallery_view])
gallery_x = feature[gseq_mask, :]
gallery_y = label[gseq_mask]
pseq_mask = np.isin(seq_type, probe_seq) & np.isin(
view, [probe_view])
probe_x = feature[pseq_mask, :]
probe_y = label[pseq_mask]
dist = cuda_dist(probe_x, gallery_x, metric)
idx = dist.sort(1)[1].cpu().numpy()
# print(p, v1, v2, "\n")
acc[p, v1, v2, :] = np.round(
np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0,
0) * 100 / dist.shape[0], 2)
result_dict = {}
for i in range(1):
msg_mgr.log_info(
'===Rank-%d (Include identical-view cases)===' % (i + 1))
msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % (
np.mean(acc[0, :, :, i]),
np.mean(acc[1, :, :, i]),
np.mean(acc[2, :, :, i]),
np.mean(acc[3, :, :, i])))
for i in range(1):
msg_mgr.log_info(
'===Rank-%d (Exclude identical-view cases)===' % (i + 1))
msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % (
de_diag(acc[0, :, :, i]),
de_diag(acc[1, :, :, i]),
de_diag(acc[2, :, :, i]),
de_diag(acc[3, :, :, i])))
result_dict["scalar/test_accuracy/CL"] = acc[0, :, :, i]
result_dict["scalar/test_accuracy/UP"] = acc[1, :, :, i]
result_dict["scalar/test_accuracy/DN"] = acc[2, :, :, i]
result_dict["scalar/test_accuracy/BG"] = acc[3, :, :, i]
np.set_printoptions(precision=2, floatmode='fixed')
for i in range(1):
msg_mgr.log_info(
'===Rank-%d of each angle (Exclude identical-view cases)===' % (i + 1))
msg_mgr.log_info('CL: {}'.format(de_diag(acc[0, :, :, i], True)))
msg_mgr.log_info('UP: {}'.format(de_diag(acc[1, :, :, i], True)))
msg_mgr.log_info('DN: {}'.format(de_diag(acc[2, :, :, i], True)))
msg_mgr.log_info('BG: {}'.format(de_diag(acc[3, :, :, i], True)))
return result_dict
def evaluate_scoliosis(data, dataset, metric='euc'):
msg_mgr = get_msg_mgr()
feature, label, class_id, view = data['embeddings'], data['labels'], data['types'], data['views']
label = np.array(label)
class_id = np.array(class_id)
# Update class_id with integer labels based on status
class_id_int = np.array([1 if status == 'positive' else 2 if status == 'neutral' else 0 for status in class_id])
print('class_id=', class_id_int)
features = np.array(feature)
c_id_int = np.argmax(features.mean(-1), axis=-1)
print('predicted_labels', c_id_int)
# Calculate sensitivity and specificity
cm = confusion_matrix(class_id_int, c_id_int, labels=[0, 1, 2])
FP = cm.sum(axis=0) - np.diag(cm)
FN = cm.sum(axis=1) - np.diag(cm)
TP = np.diag(cm)
TN = cm.sum() - (FP + FN + TP)
# Sensitivity, hit rate, recall, or true positive rate
TPR = TP / (TP + FN)
# Specificity or true negative rate
TNR = TN / (TN + FP)
accuracy = accuracy_score(class_id_int, c_id_int)
result_dict = {}
result_dict["scalar/test_accuracy/"] = accuracy
result_dict["scalar/test_sensitivity/"] = TPR
result_dict["scalar/test_specificity/"] = TNR
# Printing the sensitivity and specificity
for i, cls in enumerate(['Positive']):
print(f"{cls} Sensitivity (Recall): {TPR[i] * 100:.2f}%")
print(f"{cls} Specificity: {TNR[i] * 100:.2f}%")
print(f"Accuracy: {accuracy * 100:.2f}%")
return result_dict
Datasets
Models
