a b/src/evaluation/k_fold_validation.py
1
# K-fold cross validation for the ratio model
2
# 1. 2-8 split
3
# 2. 1-9 split
4
# 3. "Leave one out": 29 train + 1 test
5
# Compute error mean and std.
6
7
import json
8
import subprocess
9
from src.optimize import *
10
11
POSE_MODEL = 'ViTPose_large'  # ViTPose_large, ViTPose_base, OpenPose
12
13
14
def angle_difference(vector_1, vector_2):
15
    unit_vector_1 = vector_1 / np.linalg.norm(vector_1)
16
    unit_vector_2 = vector_2 / np.linalg.norm(vector_2)
17
    dot_product = np.dot(unit_vector_1, unit_vector_2)
18
    if dot_product > 1:
19
        dot_product = 1
20
    angle = np.degrees(np.arccos(dot_product))
21
    return angle
22
23
24
K = 30
25
fold = 30
26
fold_len = 1
27
test_fold = int(K / fold)
28
train_fold = K - test_fold
29
30
# collect data
31
target12_data = [[], [], []]  # list of list of np.array: [X1_list, X2_list, t2_list]
32
target1_GT_pos = []  # list of np.array
33
target2_GT_pos = []
34
target1_GT_normal = []
35
target2_GT_normal = []
36
37
target4_data = [[], [], []]
38
target4_GT_pos = []
39
target4_GT_normal = []
40
subject_names = []
41
42
for SUBJECT_NAME in os.listdir('../data'):
43
    subject_folder_path = os.path.join('../data', SUBJECT_NAME)
44
    if os.path.isfile(subject_folder_path):
45
        continue
46
    subject_names.append(SUBJECT_NAME)
47
48
    scan_pose = 'front'
49
    with open(subject_folder_path + '/' + scan_pose + '/' + POSE_MODEL + '/position_data.pickle', 'rb') as f:
50
        position_data = pickle.load(f)
51
52
    target12_data[0].append(position_data[scan_pose][0])  # X1
53
    target12_data[1].append(position_data[scan_pose][1])  # X2
54
    target12_data[2].append(position_data[scan_pose][2])  # t2
55
56
    with open(subject_folder_path + '/' + scan_pose + '/two_cam_gt.pickle', 'rb') as f:
57
        ground_truth = pickle.load(f)
58
    target1_GT_pos.append(ground_truth['target_1'])
59
    target2_GT_pos.append(ground_truth['target_2'])
60
61
    with open(subject_folder_path + '/' + scan_pose + '/two_cam_gt_normal.pickle', 'rb') as f:
62
        ground_truth = pickle.load(f)
63
    target1_GT_normal.append(ground_truth['target1_normal'])
64
    target2_GT_normal.append(ground_truth['target2_normal'])
65
66
    scan_pose = 'side'
67
    with open(subject_folder_path + '/' + scan_pose + '/' + POSE_MODEL + '/position_data.pickle', 'rb') as f:
68
        position_data = pickle.load(f)
69
70
    target4_data[0].append(position_data[scan_pose][0])  # X1
71
    target4_data[1].append(position_data[scan_pose][1])  # X2
72
    target4_data[2].append(position_data[scan_pose][2])  # t2
73
74
    with open(subject_folder_path + '/' + scan_pose + '/two_cam_gt.pickle', 'rb') as f:
75
        ground_truth = pickle.load(f)
76
    target4_GT_pos.append(ground_truth['target_4'])
77
78
    with open(subject_folder_path + '/' + scan_pose + '/two_cam_gt_normal.pickle', 'rb') as f:
79
        ground_truth = pickle.load(f)
80
    target4_GT_normal.append(ground_truth['target4_normal'])
81
82
target12_data = np.array(target12_data)  # (3,30,3,1)
83
target4_data = np.array(target4_data)  # (3,30,3,1)
84
target1_GT_pos = np.array(target1_GT_pos)  # (30, 3, 1)
85
target2_GT_pos = np.array(target2_GT_pos)
86
target4_GT_pos = np.array(target4_GT_pos)
87
target1_GT_normal = np.array(target1_GT_normal)
88
target2_GT_normal = np.array(target2_GT_normal)
89
target4_GT_normal = np.array(target4_GT_normal)
90
91
pos_err_dict = {'target1': [], 'target2': [], 'target4': []}  # store the error mean of each fold
92
normal_err_dict = {'target1': [], 'target2': [], 'target4': []}
93
94
# K-fold cross validation
95
for i in range(fold):
96
    print(i + 1, "th fold")
97
    print("=============================================================")
98
99
    # train
100
    test_idx_from = i * test_fold * fold_len
101
    test_idx_to = (i + 1) * test_fold * fold_len
102
103
    target12_data_train = np.hstack((target12_data[:, 0:test_idx_from], target12_data[:, test_idx_to:]))
104
    target4_data_train = np.hstack((target4_data[:, 0:test_idx_from], target4_data[:, test_idx_to:]))
105
106
    target1_GT_pos_test = target1_GT_pos[test_idx_from:test_idx_to]
107
    target2_GT_pos_test = target2_GT_pos[test_idx_from:test_idx_to]
108
    target4_GT_pos_test = target4_GT_pos[test_idx_from:test_idx_to]
109
    target1_GT_normal_test = target1_GT_normal[test_idx_from:test_idx_to]
110
    target2_GT_normal_test = target2_GT_normal[test_idx_from:test_idx_to]
111
    target4_GT_normal_test = target4_GT_normal[test_idx_from:test_idx_to]
112
113
    target1_GT_train = np.vstack((target1_GT_pos[0:test_idx_from], target1_GT_pos[test_idx_to:]))
114
    target2_GT_train = np.vstack((target2_GT_pos[0:test_idx_from], target2_GT_pos[test_idx_to:]))
115
    target4_GT_train = np.vstack((target4_GT_pos[0:test_idx_from], target4_GT_pos[test_idx_to:]))
116
117
    target1_ratio = optimize_front_linear(target12_data_train, target1_GT_train)
118
    target2_ratio = optimize_front_linear(target12_data_train, target2_GT_train)
119
    print("target1_ratio: \n", target1_ratio)
120
    print("target2_ratio: \n", target2_ratio)
121
122
    target4_ratio = optimize_side(target4_data_train, target4_GT_train)
123
    print("target4_ratio: \n", target4_ratio)
124
125
    # test
126
    subject_test = subject_names[test_idx_from:test_idx_to]
127
128
    tar1_pos_fold = []
129
    tar2_pos_fold = []
130
    tar4_pos_fold = []
131
132
    tar1_normal_fold = []
133
    tar2_normal_fold = []
134
    tar4_normal_fold = []
135
136
    for target1_gt_pos, target2_gt_pos, target4_gt_pos, target1_gt_normal, target2_gt_normal, target4_gt_normal, SUBJECT_NAME in zip(
137
            target1_GT_pos_test, target2_GT_pos_test,
138
            target4_GT_pos_test, target1_GT_normal_test, target2_GT_normal_test, target4_GT_normal_test, subject_test):
139
140
        print("subject: ", SUBJECT_NAME)
141
        # front
142
        subprocess.run([
143
            "python", "../compute_target.py",
144
            "--pose_model={}".format(POSE_MODEL),
145
            "--subject_name={}".format(SUBJECT_NAME),
146
            "--scan_pose={}".format('front'),
147
            "--target1_r1", str(target1_ratio[0]),
148
            "--target1_r2", str(target1_ratio[1]),
149
            "--target2_r1", str(target2_ratio[0]),
150
            "--target2_r2", str(target2_ratio[1]),
151
            "--target4_r1", str(target4_ratio[0]),
152
            "--target4_r2", str(target4_ratio[1])
153
        ])
154
        front_path = '../data/' + SUBJECT_NAME + '/front/'
155
        with open(front_path + POSE_MODEL + '/tmp_target_test.pickle', 'rb') as f:
156
            front_pos_pred = pickle.load(f)
157
        target1_pos_pred, target2_pos_pred = front_pos_pred[0], front_pos_pred[1]
158
159
        target1_pos_err = np.linalg.norm(target1_gt_pos - target1_pos_pred) * 1000
160
        target2_pos_err = np.linalg.norm(target2_gt_pos - target2_pos_pred) * 1000
161
        print("target1 pos error: ", target1_pos_err)
162
        print("target2 pos error: ", target2_pos_err)
163
        tar1_pos_fold.append(target1_pos_err)
164
        tar2_pos_fold.append(target2_pos_err)
165
166
        with open(front_path + '/pcd_coordinates.pickle', 'rb') as f:
167
            coordinates = pickle.load(f)
168
        with open(front_path + '/pcd_normals.pickle', 'rb') as f:
169
            normals = pickle.load(f)
170
        idx = np.argmin(np.square(coordinates[:, 0:2] - target1_pos_pred.squeeze()[0:2]).sum(axis=1))
171
        target1_normal_pred = normals[idx]
172
        idx = np.argmin(np.square(coordinates[:, 0:2] - target2_pos_pred.squeeze()[0:2]).sum(axis=1))
173
        target2_normal_pred = normals[idx]
174
175
        target1_normal_err = angle_difference(target1_normal_pred, target1_gt_normal)
176
        target2_normal_err = angle_difference(target2_normal_pred, target2_gt_normal)
177
        print("target1 normal error: ", target1_normal_err)
178
        print("target2 normal error: ", target2_normal_err)
179
        tar1_normal_fold.append(target1_normal_err)
180
        tar2_normal_fold.append(target2_normal_err)
181
182
        # skip outlier for openpose target 4:
183
        if POSE_MODEL == 'OpenPose' and (SUBJECT_NAME == 'charles_xu' or SUBJECT_NAME == 'jingyu_wu'):
184
            continue
185
186
        # side
187
        subprocess.run([
188
            "python", "../compute_target.py",
189
            "--pose_model={}".format(POSE_MODEL),
190
            "--subject_name={}".format(SUBJECT_NAME),
191
            "--scan_pose={}".format('side'),
192
            "--target1_r1", str(target1_ratio[0]),
193
            "--target1_r2", str(target1_ratio[1]),
194
            "--target2_r1", str(target2_ratio[0]),
195
            "--target2_r2", str(target2_ratio[1]),
196
            "--target4_r1", str(target4_ratio[0]),
197
            "--target4_r2", str(target4_ratio[1])
198
        ])
199
        side_path = '../data/' + SUBJECT_NAME + '/side/'
200
        with open(side_path + POSE_MODEL + '/tmp_target_test.pickle', 'rb') as f:
201
            side_pos_pred = pickle.load(f)
202
        target4_pos_pred = side_pos_pred[0]
203
        target4_pos_err = np.linalg.norm(target4_gt_pos - target4_pos_pred) * 1000
204
        print("target4 pos error: ", target4_pos_err)
205
        tar4_pos_fold.append(target4_pos_err)
206
207
        with open(side_path + '/pcd_coordinates.pickle', 'rb') as f:
208
            coordinates = pickle.load(f)
209
        with open(side_path + '/pcd_normals.pickle', 'rb') as f:
210
            normals = pickle.load(f)
211
        idx = np.argmin(np.square(coordinates[:, 0:2] - target4_pos_pred.squeeze()[0:2]).sum(axis=1))
212
        target4_normal_pred = normals[idx]
213
214
        target4_normal_err = angle_difference(target4_normal_pred, target4_gt_normal)
215
        print("target4 normal error: ", target4_normal_err)
216
        tar4_normal_fold.append(target4_normal_err)
217
218
    pos_err_dict['target1'].append(np.mean(tar1_pos_fold))
219
    pos_err_dict['target2'].append(np.mean(tar2_pos_fold))
220
    normal_err_dict['target1'].append(np.mean(tar1_normal_fold))
221
    normal_err_dict['target2'].append(np.mean(tar2_normal_fold))
222
    if len(tar4_pos_fold) != 0:
223
        pos_err_dict['target4'].append(np.mean(tar4_pos_fold))
224
        normal_err_dict['target4'].append(np.mean(tar4_normal_fold))
225
226
227
pos_err_dict['target1_mean'] = np.mean(pos_err_dict['target1'])
228
pos_err_dict['target1_std'] = np.std(pos_err_dict['target1'])
229
pos_err_dict['target2_mean'] = np.mean(pos_err_dict['target2'])
230
pos_err_dict['target2_std'] = np.std(pos_err_dict['target2'])
231
pos_err_dict['target4_mean'] = np.mean(pos_err_dict['target4'])
232
pos_err_dict['target4_std'] = np.std(pos_err_dict['target4'])
233
234
normal_err_dict['target1_mean'] = np.mean(normal_err_dict['target1'])
235
normal_err_dict['target1_std'] = np.std(normal_err_dict['target1'])
236
normal_err_dict['target2_mean'] = np.mean(normal_err_dict['target2'])
237
normal_err_dict['target2_std'] = np.std(normal_err_dict['target2'])
238
normal_err_dict['target4_mean'] = np.mean(normal_err_dict['target4'])
239
normal_err_dict['target4_std'] = np.std(normal_err_dict['target4'])
240
241
pos_json_object = json.dumps(pos_err_dict, indent=4)
242
with open('k_fold_validation_position_result/' + POSE_MODEL + '_one_out_err.json', 'w') as f:
243
    f.write(pos_json_object)
244
245
normal_json_object = json.dumps(normal_err_dict, indent=4)
246
with open('k_fold_validation_normal_result/' + POSE_MODEL + '_one_out_err.json', 'w') as f:
247
    f.write(normal_json_object)