Datasets
Models
Diff of
/opengait/evaluation/evaluator.py
[000000]
..
[fd9ef4]
Switch to side-by-side view
--- a +++ b/opengait/evaluation/evaluator.py @@ -0,0 +1,459 @@ +import os +from time import strftime, localtime +import numpy as np +from utils import get_msg_mgr, mkdir + +from .metric import mean_iou, cuda_dist, compute_ACC_mAP, evaluate_rank, evaluate_many +from .re_rank import re_ranking +from sklearn.metrics import confusion_matrix, accuracy_score + +def de_diag(acc, each_angle=False): + # Exclude identical-view cases + dividend = acc.shape[1] - 1. + result = np.sum(acc - np.diag(np.diag(acc)), 1) / dividend + if not each_angle: + result = np.mean(result) + return result + + +def cross_view_gallery_evaluation(feature, label, seq_type, view, dataset, metric): + '''More details can be found: More details can be found in + [A Comprehensive Study on the Evaluation of Silhouette-based Gait Recognition](https://ieeexplore.ieee.org/document/9928336). + ''' + probe_seq_dict = {'CASIA-B': {'NM': ['nm-01'], 'BG': ['bg-01'], 'CL': ['cl-01']}, + 'OUMVLP': {'NM': ['00']}} + + gallery_seq_dict = {'CASIA-B': ['nm-02', 'bg-02', 'cl-02'], + 'OUMVLP': ['01']} + + msg_mgr = get_msg_mgr() + acc = {} + mean_ap = {} + view_list = sorted(np.unique(view)) + for (type_, probe_seq) in probe_seq_dict[dataset].items(): + acc[type_] = np.zeros(len(view_list)) - 1. + mean_ap[type_] = np.zeros(len(view_list)) - 1. + for (v1, probe_view) in enumerate(view_list): + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, probe_view) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + gseq_mask = np.isin(seq_type, gallery_seq_dict[dataset]) + gallery_y = label[gseq_mask] + gallery_x = feature[gseq_mask, :] + dist = cuda_dist(probe_x, gallery_x, metric) + eval_results = compute_ACC_mAP( + dist.cpu().numpy(), probe_y, gallery_y, view[pseq_mask], view[gseq_mask]) + acc[type_][v1] = np.round(eval_results[0] * 100, 2) + mean_ap[type_][v1] = np.round(eval_results[1] * 100, 2) + + result_dict = {} + msg_mgr.log_info( + '===Cross View Gallery Evaluation (Excluded identical-view cases)===') + out_acc_str = "========= Rank@1 Acc =========\n" + out_map_str = "============= mAP ============\n" + for type_ in probe_seq_dict[dataset].keys(): + avg_acc = np.mean(acc[type_]) + avg_map = np.mean(mean_ap[type_]) + result_dict[f'scalar/test_accuracy/{type_}-Rank@1'] = avg_acc + result_dict[f'scalar/test_accuracy/{type_}-mAP'] = avg_map + out_acc_str += f"{type_}:\t{acc[type_]}, mean: {avg_acc:.2f}%\n" + out_map_str += f"{type_}:\t{mean_ap[type_]}, mean: {avg_map:.2f}%\n" + # msg_mgr.log_info(f'========= Rank@1 Acc =========') + msg_mgr.log_info(f'{out_acc_str}') + # msg_mgr.log_info(f'========= mAP =========') + msg_mgr.log_info(f'{out_map_str}') + return result_dict + +# Modified From https://github.com/AbnerHqC/GaitSet/blob/master/model/utils/evaluator.py + + +def single_view_gallery_evaluation(feature, label, seq_type, view, dataset, metric): + probe_seq_dict = {'CASIA-B': {'NM': ['nm-05', 'nm-06'], 'BG': ['bg-01', 'bg-02'], 'CL': ['cl-01', 'cl-02']}, + 'OUMVLP': {'NM': ['00']}, + 'CASIA-E': {'NM': ['H-scene2-nm-1', 'H-scene2-nm-2', 'L-scene2-nm-1', 'L-scene2-nm-2', 'H-scene3-nm-1', 'H-scene3-nm-2', 'L-scene3-nm-1', 'L-scene3-nm-2', 'H-scene3_s-nm-1', 'H-scene3_s-nm-2', 'L-scene3_s-nm-1', 'L-scene3_s-nm-2', ], + 'BG': ['H-scene2-bg-1', 'H-scene2-bg-2', 'L-scene2-bg-1', 'L-scene2-bg-2', 'H-scene3-bg-1', 'H-scene3-bg-2', 'L-scene3-bg-1', 'L-scene3-bg-2', 'H-scene3_s-bg-1', 'H-scene3_s-bg-2', 'L-scene3_s-bg-1', 'L-scene3_s-bg-2'], + 'CL': ['H-scene2-cl-1', 'H-scene2-cl-2', 'L-scene2-cl-1', 'L-scene2-cl-2', 'H-scene3-cl-1', 'H-scene3-cl-2', 'L-scene3-cl-1', 'L-scene3-cl-2', 'H-scene3_s-cl-1', 'H-scene3_s-cl-2', 'L-scene3_s-cl-1', 'L-scene3_s-cl-2'] + }, + 'SUSTech1K': {'Normal': ['01-nm'], 'Bag': ['bg'], 'Clothing': ['cl'], 'Carrying':['cr'], 'Umberalla': ['ub'], 'Uniform': ['uf'], 'Occlusion': ['oc'],'Night': ['nt'], 'Overall': ['01','02','03','04']} + } + gallery_seq_dict = {'CASIA-B': ['nm-01', 'nm-02', 'nm-03', 'nm-04'], + 'OUMVLP': ['01'], + 'CASIA-E': ['H-scene1-nm-1', 'H-scene1-nm-2', 'L-scene1-nm-1', 'L-scene1-nm-2'], + 'SUSTech1K': ['00-nm'],} + msg_mgr = get_msg_mgr() + acc = {} + view_list = sorted(np.unique(view)) + num_rank = 1 + if dataset == 'CASIA-E': + view_list.remove("270") + if dataset == 'SUSTech1K': + num_rank = 5 + view_num = len(view_list) + + for (type_, probe_seq) in probe_seq_dict[dataset].items(): + acc[type_] = np.zeros((view_num, view_num, num_rank)) - 1. + for (v1, probe_view) in enumerate(view_list): + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, probe_view) + pseq_mask = pseq_mask if 'SUSTech1K' not in dataset else np.any(np.asarray( + [np.char.find(seq_type, probe)>=0 for probe in probe_seq]), axis=0 + ) & np.isin(view, probe_view) # For SUSTech1K only + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + for (v2, gallery_view) in enumerate(view_list): + gseq_mask = np.isin(seq_type, gallery_seq_dict[dataset]) & np.isin( + view, [gallery_view]) + gseq_mask = gseq_mask if 'SUSTech1K' not in dataset else np.any(np.asarray( + [np.char.find(seq_type, gallery)>=0 for gallery in gallery_seq_dict[dataset]]), axis=0 + ) & np.isin(view, [gallery_view]) # For SUSTech1K only + gallery_y = label[gseq_mask] + gallery_x = feature[gseq_mask, :] + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.topk(num_rank, largest=False)[1].cpu().numpy() + acc[type_][v1, v2, :] = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + + result_dict = {} + msg_mgr.log_info('===Rank-1 (Exclude identical-view cases)===') + out_str = "" + for rank in range(num_rank): + out_str = "" + for type_ in probe_seq_dict[dataset].keys(): + sub_acc = de_diag(acc[type_][:,:,rank], each_angle=True) + if rank == 0: + msg_mgr.log_info(f'{type_}@R{rank+1}: {sub_acc}') + result_dict[f'scalar/test_accuracy/{type_}@R{rank+1}'] = np.mean(sub_acc) + out_str += f"{type_}@R{rank+1}: {np.mean(sub_acc):.2f}%\t" + msg_mgr.log_info(out_str) + return result_dict + + +def evaluate_indoor_dataset(data, dataset, metric='euc', cross_view_gallery=False): + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + label = np.array(label) + view = np.array(view) + + if dataset not in ('CASIA-B', 'OUMVLP', 'CASIA-E', 'SUSTech1K'): + raise KeyError("DataSet %s hasn't been supported !" % dataset) + if cross_view_gallery: + return cross_view_gallery_evaluation( + feature, label, seq_type, view, dataset, metric) + else: + return single_view_gallery_evaluation( + feature, label, seq_type, view, dataset, metric) + + +def evaluate_real_scene(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + feature, label, seq_type = data['embeddings'], data['labels'], data['types'] + label = np.array(label) + + gallery_seq_type = {'0001-1000': ['1', '2'], + "HID2021": ['0'], '0001-1000-test': ['0'], + 'GREW': ['01'], 'TTG-200': ['1']} + probe_seq_type = {'0001-1000': ['3', '4', '5', '6'], + "HID2021": ['1'], '0001-1000-test': ['1'], + 'GREW': ['02'], 'TTG-200': ['2', '3', '4', '5', '6']} + + num_rank = 20 + acc = np.zeros([num_rank]) - 1. + gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.topk(num_rank, largest=False)[1].cpu().numpy() + acc = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + msg_mgr.log_info('==Rank-1==') + msg_mgr.log_info('%.3f' % (np.mean(acc[0]))) + msg_mgr.log_info('==Rank-5==') + msg_mgr.log_info('%.3f' % (np.mean(acc[4]))) + msg_mgr.log_info('==Rank-10==') + msg_mgr.log_info('%.3f' % (np.mean(acc[9]))) + msg_mgr.log_info('==Rank-20==') + msg_mgr.log_info('%.3f' % (np.mean(acc[19]))) + return {"scalar/test_accuracy/Rank-1": np.mean(acc[0]), "scalar/test_accuracy/Rank-5": np.mean(acc[4])} + + +def GREW_submission(data, dataset, metric='euc'): + get_msg_mgr().log_info("Evaluating GREW") + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + label = np.array(label) + view = np.array(view) + gallery_seq_type = {'GREW': ['01', '02']} + probe_seq_type = {'GREW': ['03']} + gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) + probe_x = feature[pseq_mask, :] + probe_y = view[pseq_mask] + + num_rank = 20 + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.topk(num_rank, largest=False)[1].cpu().numpy() + + save_path = os.path.join( + "GREW_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") + mkdir("GREW_result") + with open(save_path, "w") as f: + f.write("videoId,rank1,rank2,rank3,rank4,rank5,rank6,rank7,rank8,rank9,rank10,rank11,rank12,rank13,rank14,rank15,rank16,rank17,rank18,rank19,rank20\n") + for i in range(len(idx)): + r_format = [int(idx) for idx in gallery_y[idx[i, 0:num_rank]]] + output_row = '{}'+',{}'*num_rank+'\n' + f.write(output_row.format(probe_y[i], *r_format)) + print("GREW result saved to {}/{}".format(os.getcwd(), save_path)) + return + + +def HID_submission(data, dataset, rerank=True, metric='euc'): + msg_mgr = get_msg_mgr() + msg_mgr.log_info("Evaluating HID") + feature, label, seq_type = data['embeddings'], data['labels'], data['views'] + label = np.array(label) + seq_type = np.array(seq_type) + probe_mask = (label == "probe") + gallery_mask = (label != "probe") + gallery_x = feature[gallery_mask, :] + gallery_y = label[gallery_mask] + probe_x = feature[probe_mask, :] + probe_y = seq_type[probe_mask] + if rerank: + feat = np.concatenate([probe_x, gallery_x]) + dist = cuda_dist(feat, feat, metric).cpu().numpy() + msg_mgr.log_info("Starting Re-ranking") + re_rank = re_ranking( + dist, probe_x.shape[0], k1=6, k2=6, lambda_value=0.3) + idx = np.argsort(re_rank, axis=1) + else: + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.cpu().sort(1)[1].numpy() + + save_path = os.path.join( + "HID_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") + mkdir("HID_result") + with open(save_path, "w") as f: + f.write("videoID,label\n") + for i in range(len(idx)): + f.write("{},{}\n".format(probe_y[i], gallery_y[idx[i, 0]])) + print("HID result saved to {}/{}".format(os.getcwd(), save_path)) + return + + +def evaluate_segmentation(data, dataset): + labels = data['mask'] + pred = data['pred'] + miou = mean_iou(pred, labels) + get_msg_mgr().log_info('mIOU: %.3f' % (miou.mean())) + return {"scalar/test_accuracy/mIOU": miou} + + +def evaluate_Gait3D(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + + features, labels, cams, time_seqs = data['embeddings'], data['labels'], data['types'], data['views'] + import json + probe_sets = json.load( + open('./datasets/Gait3D/Gait3D.json', 'rb'))['PROBE_SET'] + probe_mask = [] + for id, ty, sq in zip(labels, cams, time_seqs): + if '-'.join([id, ty, sq]) in probe_sets: + probe_mask.append(True) + else: + probe_mask.append(False) + probe_mask = np.array(probe_mask) + + # probe_features = features[:probe_num] + probe_features = features[probe_mask] + # gallery_features = features[probe_num:] + gallery_features = features[~probe_mask] + # probe_lbls = np.asarray(labels[:probe_num]) + # gallery_lbls = np.asarray(labels[probe_num:]) + probe_lbls = np.asarray(labels)[probe_mask] + gallery_lbls = np.asarray(labels)[~probe_mask] + + results = {} + msg_mgr.log_info(f"The test metric you choose is {metric}.") + dist = cuda_dist(probe_features, gallery_features, metric).cpu().numpy() + cmc, all_AP, all_INP = evaluate_rank(dist, probe_lbls, gallery_lbls) + + mAP = np.mean(all_AP) + mINP = np.mean(all_INP) + for r in [1, 5, 10]: + results['scalar/test_accuracy/Rank-{}'.format(r)] = cmc[r - 1] * 100 + results['scalar/test_accuracy/mAP'] = mAP * 100 + results['scalar/test_accuracy/mINP'] = mINP * 100 + + # print_csv_format(dataset_name, results) + msg_mgr.log_info(results) + return results + + +def evaluate_CCPG(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + + label = np.array(label) + for i in range(len(view)): + view[i] = view[i].split("_")[0] + view_np = np.array(view) + view_list = list(set(view)) + view_list.sort() + + view_num = len(view_list) + + probe_seq_dict = {'CCPG': [["U0_D0_BG", "U0_D0"], [ + "U3_D3"], ["U1_D0"], ["U0_D0_BG"]]} + + gallery_seq_dict = { + 'CCPG': [["U1_D1", "U2_D2", "U3_D3"], ["U0_D3"], ["U1_D1"], ["U0_D0"]]} + if dataset not in (probe_seq_dict or gallery_seq_dict): + raise KeyError("DataSet %s hasn't been supported !" % dataset) + num_rank = 5 + acc = np.zeros([len(probe_seq_dict[dataset]), + view_num, view_num, num_rank]) - 1. + + ap_save = [] + cmc_save = [] + minp = [] + for (p, probe_seq) in enumerate(probe_seq_dict[dataset]): + # for gallery_seq in gallery_seq_dict[dataset]: + gallery_seq = gallery_seq_dict[dataset][p] + gseq_mask = np.isin(seq_type, gallery_seq) + gallery_x = feature[gseq_mask, :] + # print("gallery_x", gallery_x.shape) + gallery_y = label[gseq_mask] + gallery_view = view_np[gseq_mask] + + pseq_mask = np.isin(seq_type, probe_seq) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + probe_view = view_np[pseq_mask] + + msg_mgr.log_info( + ("gallery length", len(gallery_y), gallery_seq, "probe length", len(probe_y), probe_seq)) + distmat = cuda_dist(probe_x, gallery_x, metric).cpu().numpy() + # cmc, ap = evaluate(distmat, probe_y, gallery_y, probe_view, gallery_view) + cmc, ap, inp = evaluate_many( + distmat, probe_y, gallery_y, probe_view, gallery_view) + ap_save.append(ap) + cmc_save.append(cmc[0]) + minp.append(inp) + + # print(ap_save, cmc_save) + + msg_mgr.log_info( + '===Rank-1 (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + cmc_save[0]*100, cmc_save[1]*100, cmc_save[2]*100, cmc_save[3]*100)) + + msg_mgr.log_info( + '===mAP (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + ap_save[0]*100, ap_save[1]*100, ap_save[2]*100, ap_save[3]*100)) + + msg_mgr.log_info( + '===mINP (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % + (minp[0]*100, minp[1]*100, minp[2]*100, minp[3]*100)) + + for (p, probe_seq) in enumerate(probe_seq_dict[dataset]): + # for gallery_seq in gallery_seq_dict[dataset]: + gallery_seq = gallery_seq_dict[dataset][p] + for (v1, probe_view) in enumerate(view_list): + for (v2, gallery_view) in enumerate(view_list): + gseq_mask = np.isin(seq_type, gallery_seq) & np.isin( + view, [gallery_view]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, [probe_view]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.sort(1)[1].cpu().numpy() + # print(p, v1, v2, "\n") + acc[p, v1, v2, :] = np.round( + np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + result_dict = {} + for i in range(1): + msg_mgr.log_info( + '===Rank-%d (Include identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + np.mean(acc[0, :, :, i]), + np.mean(acc[1, :, :, i]), + np.mean(acc[2, :, :, i]), + np.mean(acc[3, :, :, i]))) + for i in range(1): + msg_mgr.log_info( + '===Rank-%d (Exclude identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + de_diag(acc[0, :, :, i]), + de_diag(acc[1, :, :, i]), + de_diag(acc[2, :, :, i]), + de_diag(acc[3, :, :, i]))) + result_dict["scalar/test_accuracy/CL"] = acc[0, :, :, i] + result_dict["scalar/test_accuracy/UP"] = acc[1, :, :, i] + result_dict["scalar/test_accuracy/DN"] = acc[2, :, :, i] + result_dict["scalar/test_accuracy/BG"] = acc[3, :, :, i] + np.set_printoptions(precision=2, floatmode='fixed') + for i in range(1): + msg_mgr.log_info( + '===Rank-%d of each angle (Exclude identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: {}'.format(de_diag(acc[0, :, :, i], True))) + msg_mgr.log_info('UP: {}'.format(de_diag(acc[1, :, :, i], True))) + msg_mgr.log_info('DN: {}'.format(de_diag(acc[2, :, :, i], True))) + msg_mgr.log_info('BG: {}'.format(de_diag(acc[3, :, :, i], True))) + return result_dict + +def evaluate_scoliosis(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + + feature, label, class_id, view = data['embeddings'], data['labels'], data['types'], data['views'] + + label = np.array(label) + class_id = np.array(class_id) + + # Update class_id with integer labels based on status + class_id_int = np.array([1 if status == 'positive' else 2 if status == 'neutral' else 0 for status in class_id]) + print('class_id=', class_id_int) + + features = np.array(feature) + c_id_int = np.argmax(features.mean(-1), axis=-1) + print('predicted_labels', c_id_int) + + # Calculate sensitivity and specificity + cm = confusion_matrix(class_id_int, c_id_int, labels=[0, 1, 2]) + FP = cm.sum(axis=0) - np.diag(cm) + FN = cm.sum(axis=1) - np.diag(cm) + TP = np.diag(cm) + TN = cm.sum() - (FP + FN + TP) + + # Sensitivity, hit rate, recall, or true positive rate + TPR = TP / (TP + FN) + # Specificity or true negative rate + TNR = TN / (TN + FP) + accuracy = accuracy_score(class_id_int, c_id_int) + + result_dict = {} + result_dict["scalar/test_accuracy/"] = accuracy + result_dict["scalar/test_sensitivity/"] = TPR + result_dict["scalar/test_specificity/"] = TNR + + # Printing the sensitivity and specificity + for i, cls in enumerate(['Positive']): + print(f"{cls} Sensitivity (Recall): {TPR[i] * 100:.2f}%") + print(f"{cls} Specificity: {TNR[i] * 100:.2f}%") + print(f"Accuracy: {accuracy * 100:.2f}%") + + return result_dict \ No newline at end of file