Diff of /app/LeapData.py [000000] .. [4de1c7]

Switch to unified view

a b/app/LeapData.py
1
import re
2
import numpy as np
3
import pandas
4
import sys
5
6
from config.Skeleton import Skeleton
7
from config.AnybodyFirstFrame import AnybodyFirstFrame
8
from config.BasisFirstFrame import BasisFirstFrame
9
from resources.pymo.pymo.data import MocapData
10
from RotationUtil import rot2eul, get_order
11
from resources.LeapSDK.v53_python39 import Leap
12
13
14
class LeapData:
15
    """
16
    A class to convert LeapMotion frames to PyMO data structure (MocapData)
17
18
    Calculates translations (offsets) and rotation data for the joints
19
    """
20
21
    def __init__(self, channel_setting='rotation', frame_rate=0.033333, anybody_basis=True):
22
        self._skeleton = {}
23
        self._setting = Skeleton(channel_setting)
24
        self._motion_channels = []
25
        self._motions = []
26
        self._root_name = ''
27
        self.data = MocapData()
28
29
        self.first_frame = None
30
        self.anybody_first_frame = AnybodyFirstFrame()
31
        self.basis_first_frame = BasisFirstFrame()
32
        # anybody_reference = True -> Use Anybody basis from config/*.json (see AnybodyFirstFrame)
33
        # anybody_reference = False -> Use Leap Motion First Frame for basis
34
        self.anybody_basis = anybody_basis
35
        self.status = 0
36
        self._frame_rate = frame_rate
37
38
        self._skeleton = self._setting.skeleton
39
        # fill channels into skeleton in selected order (i.e. xyz)
40
        self._skeleton_apply_channels(self._setting.channel_setting)
41
        self._root_name = self._setting.root_name
42
43
        # initialize offsets for each joint
44
        for joint_name, joint_value in self._skeleton.items():
45
            joint_value['offsets'] = [0, 0, 0]
46
            for channel in joint_value['channels']:
47
                self._motion_channels.append((joint_name, channel))
48
49
    def parse(self):
50
        self.data.skeleton = self._skeleton
51
        self.data.channel_names = self._motion_channels
52
        if not self.first_frame:
53
            sys.exit("No data was recorded - will terminate now!")
54
        self.data.values = self._motion2dataframe()
55
        self.data.root_name = self._root_name
56
        self.data.framerate = self._frame_rate
57
58
        return self.data
59
60
    def _check_frame(self, frame):
61
        """
62
        Check whether frame and hand and fingers are valid, produce error when left hand is shown
63
        """
64
        status_no_hand = 1
65
        status_no_finger = 2
66
        status_left_hand = 3
67
        status_valid = 4
68
69
        if frame.hands.is_empty:
70
            if self.status != status_no_hand:
71
                print("-- No hand found. --")
72
                self.status = status_no_hand
73
            return False
74
75
        # Get the first hand
76
        hand = frame.hands[0]
77
78
        if hand.is_left:
79
            if self.status != status_left_hand:
80
                print("-- Please use your right hand. --")
81
                self.status = status_left_hand
82
            return False
83
84
        if not hand.is_right and not hand.is_valid:
85
            return False
86
87
        # Check if the hand has any fingers
88
        fingers = hand.fingers
89
        if fingers.is_empty:
90
            if self.status != status_no_finger:
91
                print("-- No valid fingers found. --")
92
                self.status = status_no_finger
93
            return False
94
95
        # frame_number = 0 if not self.first_frame else frame.id - self.first_frame.id
96
        if self.status != status_valid:
97
            print("-- Valid right hand found, recording data. --")
98
            self.status = status_valid
99
        return True
100
101
    def add_frame(self, frame):
102
        if not self._check_frame(frame):
103
            return None
104
105
        # Get the first hand
106
        hand = frame.hands[0]
107
        if not self.first_frame:
108
            self.first_frame = frame
109
            channel_values = self._get_channel_values(hand, firstframe=True)
110
            self._motions.append((0, channel_values))
111
            return
112
113
        channel_values = self._get_channel_values(hand)
114
        self._motions.append((frame.timestamp - self.first_frame.timestamp, channel_values))
115
        return frame
116
117
    def _get_channel_values(self, hand, firstframe=False):
118
        channel_values = []
119
        # export_basis = {}
120
        for joint_name, joint_value in self._skeleton.items():
121
            # motion data with rotations
122
            if joint_name == self._root_name:
123
                x_pos, y_pos, z_pos = LeapData._get_root_offset()
124
            elif joint_name == 'RightElbow':
125
                x_pos, y_pos, z_pos = LeapData._get_elbow_offset(hand)
126
            elif joint_name == 'RightHand':
127
                x_pos, y_pos, z_pos = LeapData._get_wrist_offset(hand)
128
            else:
129
                x_pos, y_pos, z_pos = LeapData._get_finger_offset(joint_name, hand)
130
131
            x_rot, y_rot, z_rot = self._calculate_euler_angles(hand, joint_name)
132
133
            x_rot *= Leap.RAD_TO_DEG
134
            y_rot *= Leap.RAD_TO_DEG
135
            z_rot *= Leap.RAD_TO_DEG
136
137
            # if joint_name == 'RightHand':
138
            #     print(x_rot, y_rot, z_rot)
139
140
            if firstframe:
141
                if self.anybody_basis:
142
                    x_pos, y_pos, z_pos = self._calculate_offset(joint_name, [x_pos, y_pos, z_pos])
143
                joint_value['offsets'] = [x_pos, y_pos, z_pos]
144
145
                # # dump the basis of leap motion bones
146
                # if 'End' not in joint_name and 'Root' not in joint_name:
147
                #     export_basis[joint_name] = np.ndarray.tolist(self._get_basis(hand, joint_name))
148
149
            for channel in joint_value['channels']:
150
                if channel == 'Xposition':
151
                    channel_values.append((joint_name, channel, x_pos))
152
                if channel == 'Yposition':
153
                    channel_values.append((joint_name, channel, y_pos))
154
                if channel == 'Zposition':
155
                    channel_values.append((joint_name, channel, z_pos))
156
                if channel == 'Xrotation':
157
                    channel_values.append((joint_name, channel, x_rot))
158
                if channel == 'Yrotation':
159
                    channel_values.append((joint_name, channel, y_rot))
160
                if channel == 'Zrotation':
161
                    channel_values.append((joint_name, channel, z_rot))
162
163
        # # dump the basis of leap motion bones
164
        # if firstframe:
165
        #     import json
166
        #     import datetime
167
        #     with open('../output/{}basis.json'.format(datetime.datetime.today().strftime('%Y%m%d_%H%M%S')), 'w') as o:
168
        #         json.dump(export_basis, o)
169
170
        return channel_values
171
172
    def _calculate_euler_angles(self, hand, joint_name):
173
        initial_hand = self.first_frame.hands[0]
174
175
        # special case for root and finger tip
176
        if joint_name == self._root_name or not self._skeleton[joint_name]['children']:
177
            return 0.0, 0.0, 0.0
178
179
        if self.anybody_basis:
180
            # compare basis to anybody basis
181
            parent_initial_basis = self._get_basis_first_frame(self._skeleton[joint_name]['parent'])
182
            initial_basis = self._get_basis_first_frame(joint_name)
183
        else:
184
            # compare basis to first frame from Leap Motion
185
            parent_initial_basis = self._get_basis(initial_hand, self._skeleton[joint_name]['parent'])
186
            initial_basis = self._get_basis(initial_hand, joint_name)
187
188
        parent_basis = self._get_basis(hand, self._skeleton[joint_name]['parent'])
189
        basis = self._get_basis(hand, joint_name)
190
191
        # if joint_name == 'RightHand':
192
        #     print(basis)
193
194
        # calculation of local rotation matrix - important!!!
195
        rot = np.matmul(
196
                np.matmul(
197
                    initial_basis, np.transpose(basis)
198
                ),
199
                np.transpose(
200
                    np.matmul(
201
                        parent_initial_basis, np.transpose(parent_basis)
202
                    )
203
                )
204
        )
205
        return rot2eul(rot)
206
207
    def _get_basis(self, hand, joint_name):
208
        if joint_name == self._root_name:
209
            return np.array([[1, 0, 0],
210
                             [0, 1, 0],
211
                             [0, 0, 1]])
212
        if joint_name == 'RightElbow':
213
            return LeapData._basismatrix(hand.arm.basis)
214
        if joint_name == 'RightHand':
215
            return LeapData._basismatrix(hand.basis)
216
217
        # else, return basis of the finger
218
        finger, bone_number = LeapData._split_key(joint_name)
219
        fingerlist = hand.fingers.finger_type(LeapData._get_finger_type(finger))
220
        bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number))
221
        return LeapData._basismatrix(bone.basis)
222
223
    def _get_basis_first_frame(self, joint_name):
224
        if joint_name == self._root_name:
225
            return np.array([[1, 0, 0],
226
                             [0, 1, 0],
227
                             [0, 0, 1]])
228
        return self.anybody_first_frame.get_basis(joint_name)
229
230
    def _calculate_offset(self, joint_name, leap_offset):
231
        if joint_name == self._root_name:
232
            return 0, 0, 0
233
234
        leap_length = np.linalg.norm(leap_offset)
235
236
        position = self._get_anybody_position(joint_name)
237
        position_parent = self._get_anybody_position(self._skeleton[joint_name]['parent'])
238
        offset_anybody = position - position_parent
239
        # make a unit vector
240
        offset_anybody = np.divide(offset_anybody, np.linalg.norm(offset_anybody))
241
        offset = offset_anybody * leap_length
242
        return offset[0], offset[1], offset[2]
243
244
    def _get_anybody_position(self, joint_name):
245
        if joint_name == self._root_name:
246
            return np.array([0, 0, 0])
247
        return self.anybody_first_frame.get_position(joint_name)
248
249
    @staticmethod
250
    def _get_root_offset():
251
        return 0, 0, 0
252
253
    @staticmethod
254
    def _get_elbow_offset(hand):
255
        arm = hand.arm
256
        return arm.elbow_position.x, arm.elbow_position.y, arm.elbow_position.z
257
258
    @staticmethod
259
    def _get_wrist_offset(hand):
260
        arm = hand.arm
261
        x_wrist = hand.wrist_position.x - arm.elbow_position.x
262
        y_wrist = hand.wrist_position.y - arm.elbow_position.y
263
        z_wrist = hand.wrist_position.z - arm.elbow_position.z
264
265
        return x_wrist, y_wrist, z_wrist
266
267
    @staticmethod
268
    def _get_finger_offset(key, hand):
269
        key, bone_number = LeapData._split_key(key)
270
271
        fingerlist = hand.fingers.finger_type(LeapData._get_finger_type(key))
272
273
        # vector between wrist and joint metacarpal proximal (length of carpals)
274
        if bone_number == 1 or ('Thumb' in key and bone_number == 2):
275
            bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number))
276
            return \
277
                bone.prev_joint.x - hand.wrist_position.x, \
278
                bone.prev_joint.y - hand.wrist_position.y, \
279
                bone.prev_joint.z - hand.wrist_position.z
280
281
        # vector for bones metacarpal, proximal, intermediate, distal
282
        bone = fingerlist[0].bone(LeapData._get_bone_type(bone_number - 1))
283
        return \
284
            bone.next_joint.x - bone.prev_joint.x, \
285
            bone.next_joint.y - bone.prev_joint.y, \
286
            bone.next_joint.z - bone.prev_joint.z
287
288
    @staticmethod
289
    def _split_key(key):
290
        key_split = re.split('(\d)', key)
291
        key = key_split[0]
292
        if key_split[-1] == '_Nub':
293
            return key, 5
294
        else:
295
            return key, int(key_split[1])
296
297
    @staticmethod
298
    def _get_finger_type(key):
299
        if key == 'RightHandThumb':
300
            return Leap.Finger.TYPE_THUMB
301
        if key == 'RightHandIndex':
302
            return Leap.Finger.TYPE_INDEX
303
        if key == 'RightHandMiddle':
304
            return Leap.Finger.TYPE_MIDDLE
305
        if key == 'RightHandRing':
306
            return Leap.Finger.TYPE_RING
307
        if key == 'RightHandPinky':
308
            return Leap.Finger.TYPE_PINKY
309
        else:
310
            raise Exception('Key ({}) did not match'.format(key))
311
312
    @staticmethod
313
    def _get_bone_type(bone_number):
314
        if bone_number == 4:
315
            return Leap.Bone.TYPE_DISTAL
316
        if bone_number == 3:
317
            return Leap.Bone.TYPE_INTERMEDIATE
318
        if bone_number == 2:
319
            return Leap.Bone.TYPE_PROXIMAL
320
        if bone_number == 1:
321
            return Leap.Bone.TYPE_METACARPAL
322
        else:
323
            raise Exception('bone number ({}) did not match'.format(bone_number))
324
325
    @staticmethod
326
    def _basismatrix(basis):
327
        return np.array([[basis.x_basis.x, basis.y_basis.x, basis.z_basis.x],
328
                        [basis.x_basis.y, basis.y_basis.y, basis.z_basis.y],
329
                        [basis.x_basis.z, basis.y_basis.z, basis.z_basis.z]])
330
331
    def _get_channels(self, joint_name, channel_setting):
332
        if '_Nub' in joint_name:
333
            return []
334
335
        channels_position = ['Xposition', 'Yposition', 'Zposition']
336
        channels_rotation = ['Xrotation', 'Yrotation', 'Zrotation']
337
338
        order = get_order()  # rotation order, i.e. xyz
339
        channels_rotation = \
340
            [channels_rotation[order[0]]] + [channels_rotation[order[1]]] + [channels_rotation[order[2]]]
341
342
        if channel_setting == 'position' or joint_name == self._root_name:
343
            return channels_position + channels_rotation
344
345
        return channels_rotation
346
347
    def _skeleton_apply_channels(self, channel_setting):
348
        for joint_name, joint_dict in self._skeleton.items():
349
            joint_dict['channels'] = self._get_channels(joint_name, channel_setting)
350
351
    def _motion2dataframe(self):
352
        """Returns all of the channels parsed from the LeapMotion sensor as a pandas DataFrame"""
353
354
        time_index = pandas.to_timedelta([f[0] for f in self._motions], unit='s')
355
        frames = [f[1] for f in self._motions]
356
        channels = np.asarray([[channel[2] for channel in frame] for frame in frames])
357
        column_names = ['%s_%s' % (c[0], c[1]) for c in self._motion_channels]
358
359
        return pandas.DataFrame(data=channels, index=time_index, columns=column_names)