|
a |
|
b/vis/inv_pendulum.py |
|
|
1 |
from .visual import CocoPart |
|
|
2 |
import numpy as np |
|
|
3 |
from helpers import * |
|
|
4 |
from default_params import * |
|
|
5 |
|
|
|
6 |
|
|
|
7 |
def match_ip(ip_set, new_ips, lstm_set, num_matched, consecutive_frames=DEFAULT_CONSEC_FRAMES): |
|
|
8 |
len_ip_set = len(ip_set) |
|
|
9 |
added = [False for _ in range(len_ip_set)] |
|
|
10 |
|
|
|
11 |
new_len_ip_set = len_ip_set |
|
|
12 |
for new_ip in new_ips: |
|
|
13 |
if not is_valid(new_ip): |
|
|
14 |
continue |
|
|
15 |
# assert valid_candidate_hist(new_ip) |
|
|
16 |
cmin = [MIN_THRESH, -1] |
|
|
17 |
for i in range(len_ip_set): |
|
|
18 |
if not added[i] and dist(last_ip(ip_set[i])[0], new_ip) < cmin[0]: |
|
|
19 |
# here add dome condition that last_ip(ip_set[0] >-5 or someting) |
|
|
20 |
cmin[0] = dist(last_ip(ip_set[i])[0], new_ip) |
|
|
21 |
cmin[1] = i |
|
|
22 |
|
|
|
23 |
if cmin[1] == -1: |
|
|
24 |
ip_set.append([None for _ in range(consecutive_frames - 1)] + [new_ip]) |
|
|
25 |
lstm_set.append([None, 0, 0, 0]) # Initial hidden state of lstm is None |
|
|
26 |
new_len_ip_set += 1 |
|
|
27 |
|
|
|
28 |
else: |
|
|
29 |
added[cmin[1]] = True |
|
|
30 |
pop_and_add(ip_set[cmin[1]], new_ip, consecutive_frames) |
|
|
31 |
|
|
|
32 |
new_matched = num_matched |
|
|
33 |
|
|
|
34 |
removed_indx = [] |
|
|
35 |
removed_match = [] |
|
|
36 |
|
|
|
37 |
for i in range(len(added)): |
|
|
38 |
if not added[i]: |
|
|
39 |
pop_and_add(ip_set[i], None, consecutive_frames) |
|
|
40 |
if ip_set[i] == [None for _ in range(consecutive_frames)]: |
|
|
41 |
if i < num_matched: |
|
|
42 |
new_matched -= 1 |
|
|
43 |
removed_match.append(i) |
|
|
44 |
|
|
|
45 |
new_len_ip_set -= 1 |
|
|
46 |
removed_indx.append(i) |
|
|
47 |
|
|
|
48 |
for i in sorted(removed_indx, reverse=True): |
|
|
49 |
ip_set.pop(i) |
|
|
50 |
lstm_set.pop() |
|
|
51 |
|
|
|
52 |
return new_matched, new_len_ip_set, removed_match |
|
|
53 |
|
|
|
54 |
|
|
|
55 |
def extend_vector(p1, p2, l): |
|
|
56 |
p1 += (p1-p2)*l/(2*np.linalg.norm((p1-p2), 2)) |
|
|
57 |
p2 -= (p1-p2)*l/(2*np.linalg.norm((p1-p2), 2)) |
|
|
58 |
return p1, p2 |
|
|
59 |
|
|
|
60 |
|
|
|
61 |
def perp(a): |
|
|
62 |
b = np.empty_like(a) |
|
|
63 |
b[0] = -a[1] |
|
|
64 |
b[1] = a[0] |
|
|
65 |
return b |
|
|
66 |
|
|
|
67 |
# line segment a given by endpoints a1, a2 |
|
|
68 |
# line segment b given by endpoints b1, b2 |
|
|
69 |
# return |
|
|
70 |
|
|
|
71 |
|
|
|
72 |
def seg_intersect(a1, a2, b1, b2): |
|
|
73 |
da = a2-a1 |
|
|
74 |
db = b2-b1 |
|
|
75 |
dp = a1-b1 |
|
|
76 |
dap = perp(da) |
|
|
77 |
denom = np.dot(dap, db) |
|
|
78 |
num = np.dot(dap, dp) |
|
|
79 |
return (num / denom.astype(float))*db + b1 |
|
|
80 |
|
|
|
81 |
|
|
|
82 |
def get_kp(kp): |
|
|
83 |
threshold1 = 5e-3 |
|
|
84 |
|
|
|
85 |
# dict of np arrays of coordinates |
|
|
86 |
inv_pend = {} |
|
|
87 |
# print(type(kp[CocoPart.LEar])) |
|
|
88 |
numx = (kp[CocoPart.LEar][2]*kp[CocoPart.LEar][0] + kp[CocoPart.LEye][2]*kp[CocoPart.LEye][0] + |
|
|
89 |
kp[CocoPart.REye][2]*kp[CocoPart.REye][0] + kp[CocoPart.REar][2]*kp[CocoPart.REar][0]) |
|
|
90 |
numy = (kp[CocoPart.LEar][2]*kp[CocoPart.LEar][1] + kp[CocoPart.LEye][2]*kp[CocoPart.LEye][1] + |
|
|
91 |
kp[CocoPart.REye][2]*kp[CocoPart.REye][1] + kp[CocoPart.REar][2]*kp[CocoPart.REar][1]) |
|
|
92 |
den = kp[CocoPart.LEar][2] + kp[CocoPart.LEye][2] + kp[CocoPart.REye][2] + kp[CocoPart.REar][2] |
|
|
93 |
|
|
|
94 |
if den < HEAD_THRESHOLD: |
|
|
95 |
inv_pend['H'] = None |
|
|
96 |
else: |
|
|
97 |
inv_pend['H'] = np.array([numx/den, numy/den]) |
|
|
98 |
|
|
|
99 |
if all([kp[CocoPart.LShoulder], kp[CocoPart.RShoulder], |
|
|
100 |
kp[CocoPart.LShoulder][2] > threshold1, kp[CocoPart.RShoulder][2] > threshold1]): |
|
|
101 |
inv_pend['N'] = np.array([(kp[CocoPart.LShoulder][0]+kp[CocoPart.RShoulder][0])/2, |
|
|
102 |
(kp[CocoPart.LShoulder][1]+kp[CocoPart.RShoulder][1])/2]) |
|
|
103 |
else: |
|
|
104 |
inv_pend['N'] = None |
|
|
105 |
|
|
|
106 |
if all([kp[CocoPart.LHip], kp[CocoPart.RHip], |
|
|
107 |
kp[CocoPart.LHip][2] > threshold1, kp[CocoPart.RHip][2] > threshold1]): |
|
|
108 |
inv_pend['B'] = np.array([(kp[CocoPart.LHip][0]+kp[CocoPart.RHip][0])/2, |
|
|
109 |
(kp[CocoPart.LHip][1]+kp[CocoPart.RHip][1])/2]) |
|
|
110 |
else: |
|
|
111 |
inv_pend['B'] = None |
|
|
112 |
|
|
|
113 |
if kp[CocoPart.LKnee] is not None and kp[CocoPart.LKnee][2] > threshold1: |
|
|
114 |
inv_pend['KL'] = np.array([kp[CocoPart.LKnee][0], kp[CocoPart.LKnee][1]]) |
|
|
115 |
else: |
|
|
116 |
inv_pend['KL'] = None |
|
|
117 |
|
|
|
118 |
if kp[CocoPart.RKnee] is not None and kp[CocoPart.RKnee][2] > threshold1: |
|
|
119 |
inv_pend['KR'] = np.array([kp[CocoPart.RKnee][0], kp[CocoPart.RKnee][1]]) |
|
|
120 |
else: |
|
|
121 |
inv_pend['KR'] = None |
|
|
122 |
|
|
|
123 |
if inv_pend['B'] is not None: |
|
|
124 |
if inv_pend['N'] is not None: |
|
|
125 |
height = np.linalg.norm(inv_pend['N'] - inv_pend['B'], 2) |
|
|
126 |
LS, RS = extend_vector(np.asarray(kp[CocoPart.LShoulder][:2]), |
|
|
127 |
np.asarray(kp[CocoPart.RShoulder][:2]), height/4) |
|
|
128 |
LB, RB = extend_vector(np.asarray(kp[CocoPart.LHip][:2]), |
|
|
129 |
np.asarray(kp[CocoPart.RHip][:2]), height/3) |
|
|
130 |
ubbox = (LS, RS, RB, LB) |
|
|
131 |
|
|
|
132 |
if inv_pend['KL'] is not None and inv_pend['KR'] is not None: |
|
|
133 |
lbbox = (LB, RB, inv_pend['KR'], inv_pend['KL']) |
|
|
134 |
else: |
|
|
135 |
lbbox = ([0, 0], [0, 0]) |
|
|
136 |
#lbbox = None |
|
|
137 |
else: |
|
|
138 |
ubbox = ([0, 0], [0, 0]) |
|
|
139 |
#ubbox = None |
|
|
140 |
if inv_pend['KL'] is not None and inv_pend['KR'] is not None: |
|
|
141 |
lbbox = (np.array(kp[CocoPart.LHip][:2]), np.array(kp[CocoPart.RHip][:2]), |
|
|
142 |
inv_pend['KR'], inv_pend['KL']) |
|
|
143 |
else: |
|
|
144 |
lbbox = ([0, 0], [0, 0]) |
|
|
145 |
#lbbox = None |
|
|
146 |
else: |
|
|
147 |
ubbox = ([0, 0], [0, 0]) |
|
|
148 |
lbbox = ([0, 0], [0, 0]) |
|
|
149 |
#ubbox = None |
|
|
150 |
#lbbox = None |
|
|
151 |
# condition = (inv_pend["H"] is None) and (inv_pend['N'] is not None and inv_pend['B'] is not None) |
|
|
152 |
# if condition: |
|
|
153 |
# print("half disp") |
|
|
154 |
|
|
|
155 |
return inv_pend, ubbox, lbbox |
|
|
156 |
|
|
|
157 |
|
|
|
158 |
def get_angle(v0, v1): |
|
|
159 |
return np.math.atan2(np.linalg.det([v0, v1]), np.dot(v0, v1)) |
|
|
160 |
|
|
|
161 |
|
|
|
162 |
def is_valid(ip): |
|
|
163 |
|
|
|
164 |
assert ip is not None |
|
|
165 |
|
|
|
166 |
ip = ip["keypoints"] |
|
|
167 |
return (ip['B'] is not None and ip['N'] is not None and ip['H'] is not None) |
|
|
168 |
|
|
|
169 |
|
|
|
170 |
def get_rot_energy(ip0, ip1): |
|
|
171 |
t = ip1["time"] - ip0["time"] |
|
|
172 |
ip0 = ip0["keypoints"] |
|
|
173 |
ip1 = ip1["keypoints"] |
|
|
174 |
m1 = 1 |
|
|
175 |
m2 = 5 |
|
|
176 |
m3 = 5 |
|
|
177 |
energy = 0 |
|
|
178 |
den = 0 |
|
|
179 |
N1 = ip1['N'] - ip1['B'] |
|
|
180 |
N0 = ip0['N'] - ip0['B'] |
|
|
181 |
d2sq = N1.dot(N1) |
|
|
182 |
w2sq = (get_angle(N0, N1)/t)**2 |
|
|
183 |
energy += m2*d2sq*w2sq |
|
|
184 |
|
|
|
185 |
den += m2*d2sq |
|
|
186 |
H1 = ip1['H'] - ip1['B'] |
|
|
187 |
H0 = ip0['H'] - ip0['B'] |
|
|
188 |
d1sq = H1.dot(H1) |
|
|
189 |
w1sq = (get_angle(H0, H1)/t)**2 |
|
|
190 |
energy += m1*d1sq*w1sq |
|
|
191 |
den += m1*d1sq |
|
|
192 |
|
|
|
193 |
energy = energy/(2*den) |
|
|
194 |
# energy = energy/2 |
|
|
195 |
return energy |
|
|
196 |
|
|
|
197 |
|
|
|
198 |
def get_angle_vertical(v): |
|
|
199 |
return np.math.atan2(-v[0], -v[1]) |
|
|
200 |
|
|
|
201 |
|
|
|
202 |
def get_gf(ip0, ip1, ip2): |
|
|
203 |
t1 = ip1["time"] - ip0["time"] |
|
|
204 |
t2 = ip2["time"] - ip1["time"] |
|
|
205 |
ip0 = ip0["keypoints"] |
|
|
206 |
ip1 = ip1["keypoints"] |
|
|
207 |
ip2 = ip2["keypoints"] |
|
|
208 |
|
|
|
209 |
m1 = 1 |
|
|
210 |
m2 = 15 |
|
|
211 |
g = 10 |
|
|
212 |
H2 = ip2['H'] - ip2['N'] |
|
|
213 |
H1 = ip1['H'] - ip1['N'] |
|
|
214 |
H0 = ip0['H'] - ip0['N'] |
|
|
215 |
d1 = np.sqrt(H1.dot(H1)) |
|
|
216 |
theta_1_plus_2_2 = get_angle_vertical(H2) |
|
|
217 |
theta_1_plus_2_1 = get_angle_vertical(H1) |
|
|
218 |
theta_1_plus_2_0 = get_angle_vertical(H0) |
|
|
219 |
# print("H: ",H0,H1,H2) |
|
|
220 |
N2 = ip2['N'] - ip2['B'] |
|
|
221 |
N1 = ip1['N'] - ip1['B'] |
|
|
222 |
N0 = ip0['N'] - ip0['B'] |
|
|
223 |
d2 = np.sqrt(N1.dot(N1)) |
|
|
224 |
# print("N: ",N0,N1,N2) |
|
|
225 |
theta_2_2 = get_angle_vertical(N2) |
|
|
226 |
theta_2_1 = get_angle_vertical(N1) |
|
|
227 |
theta_2_0 = get_angle_vertical(N0) |
|
|
228 |
#print("theta_2_2:",theta_2_2,"theta_2_1:",theta_2_1,"theta_2_0:",theta_2_0,sep=", ") |
|
|
229 |
theta_1_0 = theta_1_plus_2_0 - theta_2_0 |
|
|
230 |
theta_1_1 = theta_1_plus_2_1 - theta_2_1 |
|
|
231 |
theta_1_2 = theta_1_plus_2_2 - theta_2_2 |
|
|
232 |
|
|
|
233 |
# print("theta1: ",theta_1_0,theta_1_1,theta_1_2) |
|
|
234 |
# print("theta2: ",theta_2_0,theta_2_1,theta_2_2) |
|
|
235 |
|
|
|
236 |
theta2 = theta_2_1 |
|
|
237 |
theta1 = theta_1_1 |
|
|
238 |
|
|
|
239 |
del_theta1_0 = (get_angle(H0, H1))/t1 |
|
|
240 |
del_theta1_1 = (get_angle(H1, H2))/t2 |
|
|
241 |
|
|
|
242 |
del_theta2_0 = (get_angle(N0, N1))/t1 |
|
|
243 |
del_theta2_1 = (get_angle(N1, N2))/t2 |
|
|
244 |
# print("del_theta2_1:",del_theta2_1,"del_theta2_0:",del_theta2_0,sep=",") |
|
|
245 |
del_theta1 = 0.5 * (del_theta1_1 + del_theta1_0) |
|
|
246 |
del_theta2 = 0.5 * (del_theta2_1 + del_theta2_0) |
|
|
247 |
|
|
|
248 |
doubledel_theta1 = (del_theta1_1 - del_theta1_0) / 0.5*(t1 + t2) |
|
|
249 |
doubledel_theta2 = (del_theta2_1 - del_theta2_0) / 0.5*(t1 + t2) |
|
|
250 |
# print("doubledel_theta2:",doubledel_theta2) |
|
|
251 |
|
|
|
252 |
d1 = d1/d2 |
|
|
253 |
d2 = 1 |
|
|
254 |
# print("del_theta",del_theta1,del_theta2) |
|
|
255 |
# print("doubledel_theta",doubledel_theta1,doubledel_theta2) |
|
|
256 |
|
|
|
257 |
Q_RD1 = 0 |
|
|
258 |
Q_RD1 += m1 * d1 * doubledel_theta1 * doubledel_theta1 |
|
|
259 |
Q_RD1 += (m1*d1*d1 + m1*d1*d2*np.cos(theta1))*doubledel_theta2 |
|
|
260 |
Q_RD1 += m1*d1*d2*np.sin(theta1)*del_theta2*del_theta2 |
|
|
261 |
Q_RD1 -= m1*g*d2*np.sin(theta1+theta2) |
|
|
262 |
|
|
|
263 |
Q_RD2 = 0 |
|
|
264 |
Q_RD2 += (m1*d1*d1 + m1*d1*d2*np.cos(theta1))*doubledel_theta1 |
|
|
265 |
Q_RD2 += ((m1+m2)*d2*d2 + m1*d1*d1 + 2*m1*d1*d2*np.cos(theta1))*doubledel_theta2 |
|
|
266 |
Q_RD2 -= 2*m1*d1*d2*np.sin(theta1)*del_theta2*del_theta1 + m1*d1 * \ |
|
|
267 |
d2*np.sin(theta1)*del_theta1*del_theta1 |
|
|
268 |
Q_RD2 -= (m1 + m2)*g*d2*np.sin(theta2) + m1*g*d1*np.sin(theta1 + theta2) |
|
|
269 |
|
|
|
270 |
# print("Energy: ", Q_RD1 + Q_RD2) |
|
|
271 |
return Q_RD1 + Q_RD2 |
|
|
272 |
|
|
|
273 |
|
|
|
274 |
def get_height_bbox(ip): |
|
|
275 |
bbox = ip["box"] |
|
|
276 |
assert isinstance(bbox, np.ndarray) |
|
|
277 |
diff_box = bbox[1] - bbox[0] |
|
|
278 |
return diff_box[1] |
|
|
279 |
|
|
|
280 |
|
|
|
281 |
def get_ratio_bbox(ip): |
|
|
282 |
bbox = ip["box"] |
|
|
283 |
assert isinstance(bbox, np.ndarray) |
|
|
284 |
diff_box = bbox[1] - bbox[0] |
|
|
285 |
if diff_box[1] == 0: |
|
|
286 |
diff_box[1] += 1e5*diff_box[0] |
|
|
287 |
assert(np.any(diff_box > 0)) |
|
|
288 |
ratio = diff_box[0]/diff_box[1] |
|
|
289 |
return ratio |
|
|
290 |
|
|
|
291 |
|
|
|
292 |
def get_ratio_derivative(ip0, ip1): |
|
|
293 |
ratio_der = None |
|
|
294 |
time = ip1["time"] - ip0["time"] |
|
|
295 |
diff_box = ip1["features"]["ratio_bbox"] - ip0["features"]["ratio_bbox"] |
|
|
296 |
assert time != 0 |
|
|
297 |
ratio_der = diff_box/time |
|
|
298 |
|
|
|
299 |
return ratio_der |
|
|
300 |
|
|
|
301 |
|
|
|
302 |
def match_ip2(matched_ip_set, unmatched_ip_set, new_ips, re_matrix, gf_matrix, consecutive_frames=DEFAULT_CONSEC_FRAMES): |
|
|
303 |
len_matched_ip_set = len(matched_ip_set) |
|
|
304 |
added_matched = [False for _ in range(len_matched_ip_set)] |
|
|
305 |
len_unmatched_ip_set = len(unmatched_ip_set) |
|
|
306 |
added_unmatched = [False for _ in range(len_unmatched_ip_set)] |
|
|
307 |
for new_ip in new_ips: |
|
|
308 |
if not is_valid(new_ip): |
|
|
309 |
continue |
|
|
310 |
cmin = [MIN_THRESH, -1] |
|
|
311 |
connected_set = None |
|
|
312 |
connected_added = None |
|
|
313 |
for i in range(len_matched_ip_set): |
|
|
314 |
if not added_matched[i] and dist(last_ip(matched_ip_set[i])[0], new_ip) < cmin[0]: |
|
|
315 |
# here add dome condition that last_ip(ip_set[0] >-5 or someting) |
|
|
316 |
cmin[0] = dist(last_ip(matched_ip_set[i])[0], new_ip) |
|
|
317 |
cmin[1] = i |
|
|
318 |
connected_set = matched_ip_set |
|
|
319 |
connected_added = added_matched |
|
|
320 |
for i in range(len_unmatched_ip_set): |
|
|
321 |
if not added_unmatched[i] and dist(last_ip(unmatched_ip_set[i])[0], new_ip) < cmin[0]: |
|
|
322 |
# here add dome condition that last_ip(ip_set[0] >-5 or someting) |
|
|
323 |
cmin[0] = dist(last_ip(unmatched_ip_set[i])[0], new_ip) |
|
|
324 |
cmin[1] = i |
|
|
325 |
connected_set = unmatched_ip_set |
|
|
326 |
connected_added = added_unmatched |
|
|
327 |
|
|
|
328 |
if cmin[1] == -1: |
|
|
329 |
unmatched_ip_set.append([None for _ in range(consecutive_frames - 1)] + [new_ip]) |
|
|
330 |
# re_matrix.append([]) |
|
|
331 |
# gf_matrix.append([]) |
|
|
332 |
|
|
|
333 |
else: |
|
|
334 |
connected_added[cmin[1]] = True |
|
|
335 |
pop_and_add(connected_set[cmin[1]], new_ip, consecutive_frames) |
|
|
336 |
|
|
|
337 |
i = 0 |
|
|
338 |
while i < len(added_matched): |
|
|
339 |
if not added_matched[i]: |
|
|
340 |
pop_and_add(matched_ip_set[i], None, consecutive_frames) |
|
|
341 |
if matched_ip_set[i] == [None for _ in range(consecutive_frames)]: |
|
|
342 |
matched_ip_set.pop(i) |
|
|
343 |
# re_matrix.pop(i) |
|
|
344 |
# gf_matrix.pop(i) |
|
|
345 |
added_matched.pop(i) |
|
|
346 |
continue |
|
|
347 |
i += 1 |