|
a |
|
b/app/LeapData.py |
|
|
1 |
import re |
|
|
2 |
import numpy as np |
|
|
3 |
import pandas |
|
|
4 |
import sys |
|
|
5 |
|
|
|
6 |
from config.Skeleton import Skeleton |
|
|
7 |
from config.AnybodyFirstFrame import AnybodyFirstFrame |
|
|
8 |
from config.BasisFirstFrame import BasisFirstFrame |
|
|
9 |
from resources.pymo.pymo.data import MocapData |
|
|
10 |
from RotationUtil import rot2eul, get_order |
|
|
11 |
from resources.LeapSDK.v53_python39 import Leap |
|
|
12 |
|
|
|
13 |
|
|
|
14 |
class LeapData: |
|
|
15 |
""" |
|
|
16 |
A class to convert LeapMotion frames to PyMO data structure (MocapData) |
|
|
17 |
|
|
|
18 |
Calculates translations (offsets) and rotation data for the joints |
|
|
19 |
""" |
|
|
20 |
|
|
|
21 |
def __init__(self, channel_setting='rotation', frame_rate=0.033333, anybody_basis=True): |
|
|
22 |
self._skeleton = {} |
|
|
23 |
self._setting = Skeleton(channel_setting) |
|
|
24 |
self._motion_channels = [] |
|
|
25 |
self._motions = [] |
|
|
26 |
self._root_name = '' |
|
|
27 |
self.data = MocapData() |
|
|
28 |
|
|
|
29 |
self.first_frame = None |
|
|
30 |
self.anybody_first_frame = AnybodyFirstFrame() |
|
|
31 |
self.basis_first_frame = BasisFirstFrame() |
|
|
32 |
# anybody_reference = True -> Use Anybody basis from config/*.json (see AnybodyFirstFrame) |
|
|
33 |
# anybody_reference = False -> Use Leap Motion First Frame for basis |
|
|
34 |
self.anybody_basis = anybody_basis |
|
|
35 |
self.status = 0 |
|
|
36 |
self._frame_rate = frame_rate |
|
|
37 |
|
|
|
38 |
self._skeleton = self._setting.skeleton |
|
|
39 |
# fill channels into skeleton in selected order (i.e. xyz) |
|
|
40 |
self._skeleton_apply_channels(self._setting.channel_setting) |
|
|
41 |
self._root_name = self._setting.root_name |
|
|
42 |
|
|
|
43 |
# initialize offsets for each joint |
|
|
44 |
for joint_name, joint_value in self._skeleton.items(): |
|
|
45 |
joint_value['offsets'] = [0, 0, 0] |
|
|
46 |
for channel in joint_value['channels']: |
|
|
47 |
self._motion_channels.append((joint_name, channel)) |
|
|
48 |
|
|
|
49 |
def parse(self): |
|
|
50 |
self.data.skeleton = self._skeleton |
|
|
51 |
self.data.channel_names = self._motion_channels |
|
|
52 |
if not self.first_frame: |
|
|
53 |
sys.exit("No data was recorded - will terminate now!") |
|
|
54 |
self.data.values = self._motion2dataframe() |
|
|
55 |
self.data.root_name = self._root_name |
|
|
56 |
self.data.framerate = self._frame_rate |
|
|
57 |
|
|
|
58 |
return self.data |
|
|
59 |
|
|
|
60 |
def _check_frame(self, frame): |
|
|
61 |
""" |
|
|
62 |
Check whether frame and hand and fingers are valid, produce error when left hand is shown |
|
|
63 |
""" |
|
|
64 |
status_no_hand = 1 |
|
|
65 |
status_no_finger = 2 |
|
|
66 |
status_left_hand = 3 |
|
|
67 |
status_valid = 4 |
|
|
68 |
|
|
|
69 |
if frame.hands.is_empty: |
|
|
70 |
if self.status != status_no_hand: |
|
|
71 |
print("-- No hand found. --") |
|
|
72 |
self.status = status_no_hand |
|
|
73 |
return False |
|
|
74 |
|
|
|
75 |
# Get the first hand |
|
|
76 |
hand = frame.hands[0] |
|
|
77 |
|
|
|
78 |
if hand.is_left: |
|
|
79 |
if self.status != status_left_hand: |
|
|
80 |
print("-- Please use your right hand. --") |
|
|
81 |
self.status = status_left_hand |
|
|
82 |
return False |
|
|
83 |
|
|
|
84 |
if not hand.is_right and not hand.is_valid: |
|
|
85 |
return False |
|
|
86 |
|
|
|
87 |
# Check if the hand has any fingers |
|
|
88 |
fingers = hand.fingers |
|
|
89 |
if fingers.is_empty: |
|
|
90 |
if self.status != status_no_finger: |
|
|
91 |
print("-- No valid fingers found. --") |
|
|
92 |
self.status = status_no_finger |
|
|
93 |
return False |
|
|
94 |
|
|
|
95 |
# frame_number = 0 if not self.first_frame else frame.id - self.first_frame.id |
|
|
96 |
if self.status != status_valid: |
|
|
97 |
print("-- Valid right hand found, recording data. --") |
|
|
98 |
self.status = status_valid |
|
|
99 |
return True |
|
|
100 |
|
|
|
101 |
def add_frame(self, frame): |
|
|
102 |
if not self._check_frame(frame): |
|
|
103 |
return None |
|
|
104 |
|
|
|
105 |
# Get the first hand |
|
|
106 |
hand = frame.hands[0] |
|
|
107 |
if not self.first_frame: |
|
|
108 |
self.first_frame = frame |
|
|
109 |
channel_values = self._get_channel_values(hand, firstframe=True) |
|
|
110 |
self._motions.append((0, channel_values)) |
|
|
111 |
return |
|
|
112 |
|
|
|
113 |
channel_values = self._get_channel_values(hand) |
|
|
114 |
self._motions.append((frame.timestamp - self.first_frame.timestamp, channel_values)) |
|
|
115 |
return frame |
|
|
116 |
|
|
|
117 |
def _get_channel_values(self, hand, firstframe=False): |
|
|
118 |
channel_values = [] |
|
|
119 |
# export_basis = {} |
|
|
120 |
for joint_name, joint_value in self._skeleton.items(): |
|
|
121 |
# motion data with rotations |
|
|
122 |
if joint_name == self._root_name: |
|
|
123 |
x_pos, y_pos, z_pos = LeapData._get_root_offset() |
|
|
124 |
elif joint_name == 'RightElbow': |
|
|
125 |
x_pos, y_pos, z_pos = LeapData._get_elbow_offset(hand) |
|
|
126 |
elif joint_name == 'RightHand': |
|
|
127 |
x_pos, y_pos, z_pos = LeapData._get_wrist_offset(hand) |
|
|
128 |
else: |
|
|
129 |
x_pos, y_pos, z_pos = LeapData._get_finger_offset(joint_name, hand) |
|
|
130 |
|
|
|
131 |
x_rot, y_rot, z_rot = self._calculate_euler_angles(hand, joint_name) |
|
|
132 |
|
|
|
133 |
x_rot *= Leap.RAD_TO_DEG |
|
|
134 |
y_rot *= Leap.RAD_TO_DEG |
|
|
135 |
z_rot *= Leap.RAD_TO_DEG |
|
|
136 |
|
|
|
137 |
# if joint_name == 'RightHand': |
|
|
138 |
# print(x_rot, y_rot, z_rot) |
|
|
139 |
|
|
|
140 |
if firstframe: |
|
|
141 |
if self.anybody_basis: |
|
|
142 |
x_pos, y_pos, z_pos = self._calculate_offset(joint_name, [x_pos, y_pos, z_pos]) |
|
|
143 |
joint_value['offsets'] = [x_pos, y_pos, z_pos] |
|
|
144 |
|
|
|
145 |
# # dump the basis of leap motion bones |
|
|
146 |
# if 'End' not in joint_name and 'Root' not in joint_name: |
|
|
147 |
# export_basis[joint_name] = np.ndarray.tolist(self._get_basis(hand, joint_name)) |
|
|
148 |
|
|
|
149 |
for channel in joint_value['channels']: |
|
|
150 |
if channel == 'Xposition': |
|
|
151 |
channel_values.append((joint_name, channel, x_pos)) |
|
|
152 |
if channel == 'Yposition': |
|
|
153 |
channel_values.append((joint_name, channel, y_pos)) |
|
|
154 |
if channel == 'Zposition': |
|
|
155 |
channel_values.append((joint_name, channel, z_pos)) |
|
|
156 |
if channel == 'Xrotation': |
|
|
157 |
channel_values.append((joint_name, channel, x_rot)) |
|
|
158 |
if channel == 'Yrotation': |
|
|
159 |
channel_values.append((joint_name, channel, y_rot)) |
|
|
160 |
if channel == 'Zrotation': |
|
|
161 |
channel_values.append((joint_name, channel, z_rot)) |
|
|
162 |
|
|
|
163 |
# # dump the basis of leap motion bones |
|
|
164 |
# if firstframe: |
|
|
165 |
# import json |
|
|
166 |
# import datetime |
|
|
167 |
# with open('../output/{}basis.json'.format(datetime.datetime.today().strftime('%Y%m%d_%H%M%S')), 'w') as o: |
|
|
168 |
# json.dump(export_basis, o) |
|
|
169 |
|
|
|
170 |
return channel_values |
|
|
171 |
|
|
|
172 |
def _calculate_euler_angles(self, hand, joint_name): |
|
|
173 |
initial_hand = self.first_frame.hands[0] |
|
|
174 |
|
|
|
175 |
# special case for root and finger tip |
|
|
176 |
if joint_name == self._root_name or not self._skeleton[joint_name]['children']: |
|
|
177 |
return 0.0, 0.0, 0.0 |
|
|
178 |
|
|
|
179 |
if self.anybody_basis: |
|
|
180 |
# compare basis to anybody basis |
|
|
181 |
parent_initial_basis = self._get_basis_first_frame(self._skeleton[joint_name]['parent']) |
|
|
182 |
initial_basis = self._get_basis_first_frame(joint_name) |
|
|
183 |
else: |
|
|
184 |
# compare basis to first frame from Leap Motion |
|
|
185 |
parent_initial_basis = self._get_basis(initial_hand, self._skeleton[joint_name]['parent']) |
|
|
186 |
initial_basis = self._get_basis(initial_hand, joint_name) |
|
|
187 |
|
|
|
188 |
parent_basis = self._get_basis(hand, self._skeleton[joint_name]['parent']) |
|
|
189 |
basis = self._get_basis(hand, joint_name) |
|
|
190 |
|
|
|
191 |
# if joint_name == 'RightHand': |
|
|
192 |
# print(basis) |
|
|
193 |
|
|
|
194 |
# calculation of local rotation matrix - important!!! |
|
|
195 |
rot = np.matmul( |
|
|
196 |
np.matmul( |
|
|
197 |
initial_basis, np.transpose(basis) |
|
|
198 |
), |
|
|
199 |
np.transpose( |
|
|
200 |
np.matmul( |
|
|
201 |
parent_initial_basis, np.transpose(parent_basis) |
|
|
202 |
) |
|
|
203 |
) |
|
|
204 |
) |
|
|
205 |
return rot2eul(rot) |
|
|
206 |
|
|
|
207 |
def _get_basis(self, hand, joint_name): |
|
|
208 |
if joint_name == self._root_name: |
|
|
209 |
return np.array([[1, 0, 0], |
|
|
210 |
[0, 1, 0], |
|
|
211 |
[0, 0, 1]]) |
|
|
212 |
if joint_name == 'RightElbow': |
|
|
213 |
return LeapData._basismatrix(hand.arm.basis) |
|
|
214 |
if joint_name == 'RightHand': |
|
|
215 |
return LeapData._basismatrix(hand.basis) |
|
|
216 |
|
|
|
217 |
# else, return basis of the finger |
|
|
218 |
finger, bone_number = LeapData._split_key(joint_name) |
|
|
219 |
fingerlist = hand.fingers.finger_type(LeapData._get_finger_type(finger)) |
|
|
220 |
bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number)) |
|
|
221 |
return LeapData._basismatrix(bone.basis) |
|
|
222 |
|
|
|
223 |
def _get_basis_first_frame(self, joint_name): |
|
|
224 |
if joint_name == self._root_name: |
|
|
225 |
return np.array([[1, 0, 0], |
|
|
226 |
[0, 1, 0], |
|
|
227 |
[0, 0, 1]]) |
|
|
228 |
return self.anybody_first_frame.get_basis(joint_name) |
|
|
229 |
|
|
|
230 |
def _calculate_offset(self, joint_name, leap_offset): |
|
|
231 |
if joint_name == self._root_name: |
|
|
232 |
return 0, 0, 0 |
|
|
233 |
|
|
|
234 |
leap_length = np.linalg.norm(leap_offset) |
|
|
235 |
|
|
|
236 |
position = self._get_anybody_position(joint_name) |
|
|
237 |
position_parent = self._get_anybody_position(self._skeleton[joint_name]['parent']) |
|
|
238 |
offset_anybody = position - position_parent |
|
|
239 |
# make a unit vector |
|
|
240 |
offset_anybody = np.divide(offset_anybody, np.linalg.norm(offset_anybody)) |
|
|
241 |
offset = offset_anybody * leap_length |
|
|
242 |
return offset[0], offset[1], offset[2] |
|
|
243 |
|
|
|
244 |
def _get_anybody_position(self, joint_name): |
|
|
245 |
if joint_name == self._root_name: |
|
|
246 |
return np.array([0, 0, 0]) |
|
|
247 |
return self.anybody_first_frame.get_position(joint_name) |
|
|
248 |
|
|
|
249 |
@staticmethod |
|
|
250 |
def _get_root_offset(): |
|
|
251 |
return 0, 0, 0 |
|
|
252 |
|
|
|
253 |
@staticmethod |
|
|
254 |
def _get_elbow_offset(hand): |
|
|
255 |
arm = hand.arm |
|
|
256 |
return arm.elbow_position.x, arm.elbow_position.y, arm.elbow_position.z |
|
|
257 |
|
|
|
258 |
@staticmethod |
|
|
259 |
def _get_wrist_offset(hand): |
|
|
260 |
arm = hand.arm |
|
|
261 |
x_wrist = hand.wrist_position.x - arm.elbow_position.x |
|
|
262 |
y_wrist = hand.wrist_position.y - arm.elbow_position.y |
|
|
263 |
z_wrist = hand.wrist_position.z - arm.elbow_position.z |
|
|
264 |
|
|
|
265 |
return x_wrist, y_wrist, z_wrist |
|
|
266 |
|
|
|
267 |
@staticmethod |
|
|
268 |
def _get_finger_offset(key, hand): |
|
|
269 |
key, bone_number = LeapData._split_key(key) |
|
|
270 |
|
|
|
271 |
fingerlist = hand.fingers.finger_type(LeapData._get_finger_type(key)) |
|
|
272 |
|
|
|
273 |
# vector between wrist and joint metacarpal proximal (length of carpals) |
|
|
274 |
if bone_number == 1 or ('Thumb' in key and bone_number == 2): |
|
|
275 |
bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number)) |
|
|
276 |
return \ |
|
|
277 |
bone.prev_joint.x - hand.wrist_position.x, \ |
|
|
278 |
bone.prev_joint.y - hand.wrist_position.y, \ |
|
|
279 |
bone.prev_joint.z - hand.wrist_position.z |
|
|
280 |
|
|
|
281 |
# vector for bones metacarpal, proximal, intermediate, distal |
|
|
282 |
bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number - 1)) |
|
|
283 |
return \ |
|
|
284 |
bone.next_joint.x - bone.prev_joint.x, \ |
|
|
285 |
bone.next_joint.y - bone.prev_joint.y, \ |
|
|
286 |
bone.next_joint.z - bone.prev_joint.z |
|
|
287 |
|
|
|
288 |
@staticmethod |
|
|
289 |
def _split_key(key): |
|
|
290 |
key_split = re.split('(\d)', key) |
|
|
291 |
key = key_split[0] |
|
|
292 |
if key_split[-1] == '_Nub': |
|
|
293 |
return key, 5 |
|
|
294 |
else: |
|
|
295 |
return key, int(key_split[1]) |
|
|
296 |
|
|
|
297 |
@staticmethod |
|
|
298 |
def _get_finger_type(key): |
|
|
299 |
if key == 'RightHandThumb': |
|
|
300 |
return Leap.Finger.TYPE_THUMB |
|
|
301 |
if key == 'RightHandIndex': |
|
|
302 |
return Leap.Finger.TYPE_INDEX |
|
|
303 |
if key == 'RightHandMiddle': |
|
|
304 |
return Leap.Finger.TYPE_MIDDLE |
|
|
305 |
if key == 'RightHandRing': |
|
|
306 |
return Leap.Finger.TYPE_RING |
|
|
307 |
if key == 'RightHandPinky': |
|
|
308 |
return Leap.Finger.TYPE_PINKY |
|
|
309 |
else: |
|
|
310 |
raise Exception('Key ({}) did not match'.format(key)) |
|
|
311 |
|
|
|
312 |
@staticmethod |
|
|
313 |
def _get_bone_type(bone_number): |
|
|
314 |
if bone_number == 4: |
|
|
315 |
return Leap.Bone.TYPE_DISTAL |
|
|
316 |
if bone_number == 3: |
|
|
317 |
return Leap.Bone.TYPE_INTERMEDIATE |
|
|
318 |
if bone_number == 2: |
|
|
319 |
return Leap.Bone.TYPE_PROXIMAL |
|
|
320 |
if bone_number == 1: |
|
|
321 |
return Leap.Bone.TYPE_METACARPAL |
|
|
322 |
else: |
|
|
323 |
raise Exception('bone number ({}) did not match'.format(bone_number)) |
|
|
324 |
|
|
|
325 |
@staticmethod |
|
|
326 |
def _basismatrix(basis): |
|
|
327 |
return np.array([[basis.x_basis.x, basis.y_basis.x, basis.z_basis.x], |
|
|
328 |
[basis.x_basis.y, basis.y_basis.y, basis.z_basis.y], |
|
|
329 |
[basis.x_basis.z, basis.y_basis.z, basis.z_basis.z]]) |
|
|
330 |
|
|
|
331 |
def _get_channels(self, joint_name, channel_setting): |
|
|
332 |
if '_Nub' in joint_name: |
|
|
333 |
return [] |
|
|
334 |
|
|
|
335 |
channels_position = ['Xposition', 'Yposition', 'Zposition'] |
|
|
336 |
channels_rotation = ['Xrotation', 'Yrotation', 'Zrotation'] |
|
|
337 |
|
|
|
338 |
order = get_order() # rotation order, i.e. xyz |
|
|
339 |
channels_rotation = \ |
|
|
340 |
[channels_rotation[order[0]]] + [channels_rotation[order[1]]] + [channels_rotation[order[2]]] |
|
|
341 |
|
|
|
342 |
if channel_setting == 'position' or joint_name == self._root_name: |
|
|
343 |
return channels_position + channels_rotation |
|
|
344 |
|
|
|
345 |
return channels_rotation |
|
|
346 |
|
|
|
347 |
def _skeleton_apply_channels(self, channel_setting): |
|
|
348 |
for joint_name, joint_dict in self._skeleton.items(): |
|
|
349 |
joint_dict['channels'] = self._get_channels(joint_name, channel_setting) |
|
|
350 |
|
|
|
351 |
def _motion2dataframe(self): |
|
|
352 |
"""Returns all of the channels parsed from the LeapMotion sensor as a pandas DataFrame""" |
|
|
353 |
|
|
|
354 |
time_index = pandas.to_timedelta([f[0] for f in self._motions], unit='s') |
|
|
355 |
frames = [f[1] for f in self._motions] |
|
|
356 |
channels = np.asarray([[channel[2] for channel in frame] for frame in frames]) |
|
|
357 |
column_names = ['%s_%s' % (c[0], c[1]) for c in self._motion_channels] |
|
|
358 |
|
|
|
359 |
return pandas.DataFrame(data=channels, index=time_index, columns=column_names) |