#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Fri Nov 15 12:53:26 2019 @author: 55129822 """ ''' consider tracked? ''' import numpy as np import pandas as pd import os from scipy import signal from scipy import stats from scipy.spatial.transform import Rotation as R import matplotlib.pyplot as plt from matplotlib.patches import Ellipse import matplotlib.transforms as transforms import matplotlib.image as mpl_image from PIL import Image as pil_image from mpl_toolkits.mplot3d import Axes3D import os import sys from tqdm import tqdm from enum import Enum import re from sway_utils import metrics as sm import csv from sklearn import preprocessing import scipy.misc #%% class SkeletonJoints(Enum): SPINEBASE = 0 SPINEMID = 1 NECK = 2 HEAD = 3 SHOULDERLEFT = 4 ELBOWLEFT = 5 WRISTLEFT = 6 HANDLEFT = 7 SHOULDERRIGHT = 8 ELBOWRIGHT = 9 WRISTRIGHT = 10 HANDRIGHT = 11 HIPLEFT = 12 KNEELEFT = 13 ANKLELEFT = 14 FOOTLEFT = 15 HIPRIGHT = 16 KNEERIGHT = 17 ANKLERIGHT = 18 FOOTRIGHT = 19 SPINESHOULDER = 20 HANDTIPLEFT = 21 THUMBLEFT = 22 HANDTIPRIGHT = 23 THUMBRIGHT = 24 COM = 25 #HEELLEFT = 26 #HEELRIGHT = 27 class SkeletonJoints_no_hands(Enum): SPINEBASE = 0 SPINEMID = 1 NECK = 2 #HEAD = 3 SHOULDERLEFT = 4 ELBOWLEFT = 5 WRISTLEFT = 6 #HANDLEFT = 7 SHOULDERRIGHT = 8 ELBOWRIGHT = 9 WRISTRIGHT = 10 #HANDRIGHT = 11 HIPLEFT = 12 KNEELEFT = 13 ANKLELEFT = 14 #FOOTLEFT = 15 HIPRIGHT = 16 KNEERIGHT = 17 ANKLERIGHT = 18 #FOOTRIGHT = 19 SPINESHOULDER = 20 #HANDTIPLEFT = 21 #THUMBLEFT = 22 #HANDTIPRIGHT = 23 #THUMBRIGHT = 24 COM = 25 #HEELLEFT = 26 #HEELRIGHT = 27 class HierarchicalSkeletonJoints(Enum): COM = 25 HEAD = 3 NECK = 2 SPINESHOULDER = 20 SPINEMID = 1 SPINEBASE = 0 SHOULDERLEFT = 4 SHOULDERRIGHT = 8 ELBOWLEFT = 5 ELBOWRIGHT = 9 WRISTLEFT = 6 WRISTRIGHT = 10 HIPLEFT = 12 HIPRIGHT = 16 KNEELEFT = 13 KNEERIGHT = 17 ANKLELEFT = 14 ANKLERIGHT = 18 FOOTLEFT = 15 FOOTRIGHT = 19 HANDLEFT = 7 HANDTIPLEFT = 21 THUMBLEFT = 22 HANDRIGHT = 11 HANDTIPRIGHT = 23 THUMBRIGHT = 24 class SkeletonJointAngles(Enum): BODY_COM_ANGLE = 1 BODY_LEAN_ANGLE = 2 KNEELEFT_ANGLE = 3 KNEERIGHT_ANGLE = 4 HIPLEFT_ANGLE = 5 HIPRIGHT_ANGLE = 6 ELBOWLEFT_ANGLE = 7 ELBOWLRIGHT_ANGLE = 8 ARMPITLEFT_ANGLE = 9 ARMPITROIGHT_ANGLE = 10 ANKLELEFT_ANGLE = 11 ANKLELRIGHT_ANGLE = 12 class BodySegments(Enum): #Torso HEAD_NECK = [SkeletonJoints.HEAD.value, SkeletonJoints.NECK.value] NECK_SPINESHOULDER = [SkeletonJoints.NECK.value, SkeletonJoints.SPINESHOULDER.value] SPINESHOULDER_SPINEMID = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SPINEMID.value] SPINEMID_SPINEBASE = [SkeletonJoints.SPINEMID.value, SkeletonJoints.SPINEBASE.value] #Left arm SPINESHOULDER_SHOULDERLEFT = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERLEFT.value] SHOULDERLEFT_ELBOWLEFT = [SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.ELBOWLEFT.value] ELBOWLEFT_WRISTLEFT = [SkeletonJoints.ELBOWLEFT.value, SkeletonJoints.WRISTLEFT.value] WRISTLEFT_HANDLEFT = [SkeletonJoints.WRISTLEFT.value, SkeletonJoints.HANDLEFT.value] HANDLEFT_THUMPLEFT = [SkeletonJoints.HANDLEFT.value, SkeletonJoints.THUMBLEFT.value] HANDLEFT_HANDTIPLEFT = [SkeletonJoints.HANDLEFT.value, SkeletonJoints.HANDTIPLEFT.value] #Right arm SPINESHOULDER_SHOULDERRIGHT = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERRIGHT.value] SHOULDERRIGHT_ELBOWRIGHT = [SkeletonJoints.SHOULDERRIGHT.value, SkeletonJoints.ELBOWRIGHT.value] ELBOWRIGHT_WRISTRIGHT = [SkeletonJoints.ELBOWRIGHT.value, SkeletonJoints.WRISTRIGHT.value] WRISTRIGHT_HANDRIGHT = [SkeletonJoints.WRISTRIGHT.value, SkeletonJoints.HANDRIGHT.value] HANDRIGHT_THUMPRIGHT = [SkeletonJoints.HANDRIGHT.value, SkeletonJoints.THUMBRIGHT.value] HANDRIGHT_HANDTIPRIGHT = [SkeletonJoints.HANDRIGHT.value, SkeletonJoints.HANDTIPRIGHT.value] #Left leg SPINEBASE_HIPLEFT = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPLEFT.value] HIPLEFT_KNEELEFT = [SkeletonJoints.HIPLEFT.value, SkeletonJoints.KNEELEFT.value] KNEELEFT_ANKLELEFT = [SkeletonJoints.KNEELEFT.value, SkeletonJoints.ANKLELEFT.value] ANKLELEFT_FOOTLEFT = [SkeletonJoints.ANKLELEFT.value, SkeletonJoints.FOOTLEFT.value] #Right leg SPINEBASE_HIPRIGHT = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPRIGHT.value] HIPRIGHT_KNEERIGHT = [SkeletonJoints.HIPRIGHT.value, SkeletonJoints.KNEERIGHT.value] KNEERIGHT_ANKLERIGHT = [SkeletonJoints.KNEERIGHT.value, SkeletonJoints.ANKLERIGHT.value] ANKLERIGHT_FOOTRIGHT = [SkeletonJoints.ANKLERIGHT.value, SkeletonJoints.FOOTRIGHT.value] HEAD_SPINE_BASE = [SkeletonJoints.HEAD.value, SkeletonJoints.SPINEBASE.value] SHOULDERKEFT_WRISTLEFT = [SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.WRISTLEFT.value] HIPLEFT_ANKLELEFT = [SkeletonJoints.HIPLEFT.value, SkeletonJoints.ANKLELEFT.value] class HierarchyForBodySegments(Enum): HEAD = 3 NECK = 2 SPINESHOULDER = 20 SPINEMID = 1 SPINEBASE = 0 SHOULDERLEFT = 4 SHOULDERRIGHT = 8 ELBOWLEFT = 5 ELBOWRIGHT = 9 WRISTLEFT = 6 WRISTRIGHT = 10 HIPLEFT = 12 HIPRIGHT = 16 KNEELEFT = 13 KNEERIGHT = 17 ANKLELEFT = 14 ANKLERIGHT = 18 FOOTLEFT = 15 FOOTRIGHT = 19 HANDLEFT = 7 HANDTIPLEFT = 21 THUMBLEFT = 22 HANDRIGHT = 11 HANDTIPRIGHT = 23 THUMBRIGHT = 24 class BodyParts(Enum): TORSO = [SkeletonJoints.HEAD, SkeletonJoints.NECK, SkeletonJoints.SPINESHOULDER, SkeletonJoints.SPINEMID, SkeletonJoints.SPINEBASE] ARM_LEFT = [SkeletonJoints.SHOULDERLEFT, SkeletonJoints.ELBOWLEFT, SkeletonJoints.WRISTLEFT, SkeletonJoints.HANDLEFT, SkeletonJoints.THUMBLEFT, SkeletonJoints.HANDTIPLEFT] ARM_RIGHT = [SkeletonJoints.SHOULDERRIGHT, SkeletonJoints.ELBOWRIGHT, SkeletonJoints.WRISTRIGHT, SkeletonJoints.HANDRIGHT, SkeletonJoints.THUMBRIGHT, SkeletonJoints.HANDTIPRIGHT] LEG_LEFT = [SkeletonJoints.HIPLEFT, SkeletonJoints.KNEELEFT, SkeletonJoints.ANKLELEFT, SkeletonJoints.FOOTLEFT] LEG_RIGHT = [SkeletonJoints.HIPRIGHT, SkeletonJoints.KNEERIGHT, SkeletonJoints.ANKLERIGHT, SkeletonJoints.FOOTRIGHT] class BodySections(Enum): TORSO = BodyParts.TORSO ARMS = [BodyParts.ARM_LEFT, BodyParts.ARM_RIGHT] LEGS = [BodyParts.LEG_LEFT, BodyParts.LEG_RIGHT] class ScaleBodyParts(Enum): TORSO = ['HEAD', 'NECK', 'SPINESHOULDER', 'SPINEMID', 'SPINEBASE'] ARMS = ['SHOULDERLEFT', 'EBOWLEFT', 'WRISTLEFT', 'HANDLEFT', 'THUMBLEFT', 'HANDTIPLEFT', 'SHOULDERRIGHT', 'ELBOWRIGHT', 'WRISTRIGHT', 'HANDRIGHT', 'THUMBRIGHT', 'HANDTIPRIGHT'] LEGS = ['HIPLEFT', 'KNEELEFT', 'ANKLELEFT', 'FOOTLEFT', 'HIPRIGHT', 'KNEERIGHT', 'ANKLERIGHT', 'FOOTRIGHT'] class WalkedSkelAngles(Enum): ''' inspired by: A. Vakanski, H. P. Jun, D. Paul, and R. Baker, “A data set of human body movements for physical rehabilitation exercises,” Data, vol. 3, no. 1, 2018. NB no head tip and no tip ''' #Torso SPINEBASE_SPINEMID = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.SPINEMID.value] SPINEMID_SPINESHOULDER = [SkeletonJoints.SPINEMID.value, SkeletonJoints.SPINESHOULDER.value] SPINESHOULDER_NECK = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.NECK.value] NECK_HEAD = [SkeletonJoints.NECK.value, SkeletonJoints.HEAD.value] #Left upper SPINESHOULDER_SHOULDERLEFT = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERLEFT.value] SHOULDERLEFT_ELBOWLEFT = [SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.ELBOWLEFT.value] ELBOWLEFT_WRISTLEFT = [SkeletonJoints.ELBOWLEFT.value, SkeletonJoints.WRISTLEFT.value] WRISTLEFT_HANDLEFT = [SkeletonJoints.WRISTLEFT.value, SkeletonJoints.HANDLEFT.value] #Right upper SPINESHOULDER_SHOULDERRIGHT = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERRIGHT.value] SHOULDERRIGHT_ELBOWRIGHT = [SkeletonJoints.SHOULDERRIGHT.value, SkeletonJoints.ELBOWRIGHT.value] ELBOWRIGHT_WRISTRIGHT = [SkeletonJoints.ELBOWRIGHT.value, SkeletonJoints.WRISTRIGHT.value] WRISTRIGHT_HANDRIGHT = [SkeletonJoints.WRISTRIGHT.value, SkeletonJoints.HANDRIGHT.value] #Left lower SPINEBASE_HIPLEFT = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPLEFT.value] HIPLEFT_KNEELEFT = [SkeletonJoints.HIPLEFT.value, SkeletonJoints.KNEELEFT.value] KNEELEFT_ANKLELEFT = [SkeletonJoints.KNEELEFT.value, SkeletonJoints.ANKLELEFT.value] ANKLELEFT_FOOTLEFT = [SkeletonJoints.ANKLELEFT.value, SkeletonJoints.FOOTLEFT.value] #Right lower SPINEBASE_HIPRIGHT = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPRIGHT.value] HIPRIGHT_KNEERIGHT = [SkeletonJoints.HIPRIGHT.value, SkeletonJoints.KNEERIGHT.value] KNEERIGHT_ANKLERIGHT = [SkeletonJoints.KNEERIGHT.value, SkeletonJoints.ANKLERIGHT.value] ANKLERIGHT_FOOTRIGHT = [SkeletonJoints.ANKLERIGHT.value, SkeletonJoints.FOOTRIGHT.value] class WalkedSkelAnglesIn3s(Enum): ''' Walk skeleton and get angles ''' #Torso SPINEMID_a = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.SPINEMID.value, SkeletonJoints.SPINESHOULDER.value] SPINESHOULDER_a = [SkeletonJoints.SPINEMID.value, SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.NECK.value] NECK_a = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.NECK.value, SkeletonJoints.HEAD.value] #Upper body ARMPITLEFT_a = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.ELBOWLEFT.value] ELBOWLEFT_a = [SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.ELBOWLEFT.value, SkeletonJoints.WRISTLEFT.value] ARMPITRIGHT_a = [SkeletonJoints.SPINESHOULDER.value, SkeletonJoints.SHOULDERRIGHT.value, SkeletonJoints.ELBOWRIGHT.value] ELBOWRIGHT_a = [SkeletonJoints.SHOULDERLEFT.value, SkeletonJoints.ELBOWRIGHT.value, SkeletonJoints.WRISTRIGHT.value] #Lower body HIPLEFT_a = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPLEFT.value, SkeletonJoints.KNEELEFT.value] KNEELEFT_a = [SkeletonJoints.HIPLEFT.value, SkeletonJoints.KNEELEFT.value, SkeletonJoints.ANKLELEFT.value] HIPRIGHT_a = [SkeletonJoints.SPINEBASE.value, SkeletonJoints.HIPRIGHT.value, SkeletonJoints.KNEERIGHT.value] KNEERIGHT_a = [SkeletonJoints.HIPLEFT.value, SkeletonJoints.KNEERIGHT.value, SkeletonJoints.ANKLERIGHT.value] class RefernceTorsoJoints_HEAD(Enum): HEAD = SkeletonJoints.HEAD.value SHOULDERLEFT = SkeletonJoints.SHOULDERLEFT.value SHOULDERRIGHT = SkeletonJoints.SHOULDERRIGHT.value HIPLEFT = SkeletonJoints.HIPLEFT.value HIPRIGHT = SkeletonJoints.HIPRIGHT.value class RefernceTorsoJoints_NECK(Enum): NECK = SkeletonJoints.NECK.value SHOULDERLEFT = SkeletonJoints.SHOULDERLEFT.value SHOULDERRIGHT = SkeletonJoints.SHOULDERRIGHT.value HIPLEFT = SkeletonJoints.HIPLEFT.value HIPRIGHT = SkeletonJoints.HIPRIGHT.value class RefernceTorsoJoints_COM(Enum): COM = SkeletonJoints.COM.value X = 0 Y = 1 Z = 2 #%% ''' General utilities ''' def walkFileSystem(filePath): root = [] dirs = [] files = [] for root, dirs, files in os.walk(filePath): break d = '' for d in dirs: if 'remov' in d.lower(): dirs.remove(d) d = '' for d in dirs: if 'up-and-go' in d.lower(): dirs.remove(d) d = '' for d in dirs: if '3m' in d.lower(): dirs.remove(d) d = '' for d in dirs: if 'toe' in d.lower(): dirs.remove(d) d = '' for d in dirs: if 'dual' in d.lower(): dirs.remove(d) return root, dirs, files #%% class KinectRecording: ''' Class that represents a single kinect recording ''' _root_path = '' _skel_root_path = '' _skel_file_root = '' _dataset_prefix = '' _movement = '' _part_id = 0 _labels = [] _ref_spine_base = [] #spine base of first skel frame _ref_skel_frame = [] #whole of first skel frame _frame_count = -1 _load_from_cache = False _cached_stacked_raw_XYZ_file_path = '' _cached_skeletons_path = '' _skel_scale = [] _torso_scale = 0 #_ref_neck_length = 0.15 _scale_skeletons = False '''' Values from files skeletons: list of pandas DataFrames, representing text files from kinect: [features, SkeletonJoints] tipically, [6,26] fetures : Indes, Tracked, X, Y, Z, px_X, px_Y ''' skeletons = [] raw_XYZ_values = [] '''' stacked XYZ values are flattend versions of the raw and filteterd kinect recordings [XYZ, SkeletonJoints, #frames] typically,[3, 26, 600] ''' stacked_raw_XYZ_values = [] stacked_filtered_XYZ_values = [] '''' stacked angle values represent the key angles of the filtered XYZ values [fetures, #frames] typically,[3, 600] featurs: COM_angle, knee_angle, hip_angle_side, hip_angle_front, lean_angle ''' stacked_raw_angle_values = [] stacked_filtered_angle_vlaues = [] ''' get sway metrics from file ''' sway_metrics = [] ''' calulate and store local inter-joint coordination pattern (IJCP) ''' IJCP_Dist = [] IJCP_Velo = [] ''' Calulate and store smc features ''' smc_features = [] ''' calulate and store Walling Skel Angles ''' walked_skel_angles = [] ''' Calulate and sotre Cosine Distance and Normalised Magnitude from Q. Ke, S. An, M. Bennamoun, F. Sohel, and F. Boussaid, “SkeletonNet: Mining Deep Part Features for 3-D Action Recognition,” IEEE Signal Process. Lett., vol. 24, no. 6, pp. 731–735, Jun. 2017. ''' cosine_distance = [] normalised_magnitude = [] def __init__(self, skel_root_path, dataset_prefix, movement, part_id, labels=[], scale_skeletons=False): self._root_path = str.replace(skel_root_path, '/skel', '') self._root_path = str.replace(self._root_path, '\skel', '') self._dataset_prefix = dataset_prefix self._skel_root_path = skel_root_path self._movement = movement self._part_id = part_id self._labels = labels self._scale_skeletons = scale_skeletons self.load_skeletons(skel_root_path) self.load_sway_metrics() def load_sway_metrics(self): sway_metric_path = os.path.join(self._root_path, 'sway_metrics.csv') if os.path.exists(sway_metric_path): self.sway_metrics = pd.read_csv(sway_metric_path) def save_stacked_raw_XYZ(self): if not os.path.exists(self._cached_stacked_raw_XYZ_file_path): np.save(self._cached_stacked_raw_XYZ_file_path, self.stacked_raw_XYZ_values) def save_skeletons(self): if not os.path.exists(self._cached_skeletons_path): #np.save(self._cached_skeletons_path, self.skeletons) pd.concat(self.skeletons).to_csv(self._cached_skeletons_path) def load_skeletons(self, skel_root_path): #If chahed file exists, load #Load scaled or none-scaled versions if self._scale_skeletons: cached_stacked_raw_XYZ_file_name = (self._dataset_prefix + str(self._part_id) + '_' + 'cached_stacked_raw_XYZ_scaled_' + #'cached_stacked_ankle_raw_XYZ_scaled_' + self._movement + '.npy') cached_skeletons_file_name = (self._dataset_prefix + str(self._part_id) + '_' + 'cached_skeletons_scaled_' + #'cached_skeletons_ankle_scaled_' + self._movement + '.csv') else: cached_stacked_raw_XYZ_file_name = (self._dataset_prefix + str(self._part_id) + '_' + 'cached_stacked_raw_XYZ_' + self._movement + '.npy') cached_skeletons_file_name = (self._dataset_prefix + str(self._part_id) + '_' + 'cached_skeletons' + self._movement + '.csv') self._cached_stacked_raw_XYZ_file_path = os.path.join(self._root_path, cached_stacked_raw_XYZ_file_name) self._cached_skeletons_path = os.path.join(self._root_path, cached_skeletons_file_name) if os.path.exists(self._cached_stacked_raw_XYZ_file_path): self.stacked_raw_XYZ_values = np.load(self._cached_stacked_raw_XYZ_file_path) self._load_from_cache = True #print('loading:', self._cached_stacked_raw_XYZ_file_path, '\n') if os.path.exists(self._cached_skeletons_path): self.skeletons = pd.read_csv(self._cached_skeletons_path) #self._load_from_cache = True #print('loading:', self._cached_skeletons_path, '\n') #else calulate stuff else: root, dirs, skel_files = walkFileSystem(skel_root_path) skel_files.sort() if len(skel_files) == 0: print('Skel files for:', self._cached_stacked_raw_XYZ_file_path) for skelfile in tqdm(skel_files): #for skelfile in skel_files: skel_file_path = os.path.join(root, skelfile) _skel_frame, _raw_XYZ = self._load_skel_file(skel_file_path) '''skels are now normailesed and com added ''' self.skeletons.append(_skel_frame) self.raw_XYZ_values.append(_raw_XYZ) #dont include first frame, having a skel base of 0,0,0 can cause problems if self._frame_count > 0: if len(self.stacked_raw_XYZ_values) == 0: self.stacked_raw_XYZ_values = _raw_XYZ else: self.stacked_raw_XYZ_values = np.dstack([self.stacked_raw_XYZ_values, _raw_XYZ]) #Save self.save_stacked_raw_XYZ() self.save_skeletons() #now calulate features self.stacked_filtered_XYZ_values = self.filter_joint_sequences(self.stacked_raw_XYZ_values) return def filter_joint_sequences(self, noisy_raw_XYZ, N=2, fc=10, fs=30): stacked_filtered_XYZ_values = [] filtered_X = [] filtered_Z = [] filtered_Z = [] for joint in SkeletonJoints: joint_number = joint.value #or 'WRIST' in joint.name if 'HAND' in joint.name or 'THUMB' in joint.name or 'FOOT' in joint.name or 'ANKLE' in joint.name: _N=2 _fc=15 _fs=fs else: _N=N _fc=fc _fs=fs X, Y, Z = sm.filter_signal(noisy_raw_XYZ[0, joint_number, :], noisy_raw_XYZ[1, joint_number, :], noisy_raw_XYZ[2, joint_number, :], N=_N, fc=_fc, fs=_fs) if len(filtered_X) == 0: filtered_X = X filtered_Y = Y filtered_Z = Z else: filtered_X = np.dstack([filtered_X, X]) filtered_Y = np.dstack([filtered_Y, Y]) filtered_Z = np.dstack([filtered_Z, Z]) filtered_X = np.transpose(filtered_X) filtered_Y = np.transpose(filtered_Y) filtered_Z = np.transpose(filtered_Z) stacked_filtered_XYZ_values = np.stack([filtered_X[:,:,0], filtered_Y[:,:,0], filtered_Z[:,:,0]]) return stacked_filtered_XYZ_values def calulate_CD_and_NM(self, rebuild=False, save_pngs=False): scaled = '' if self._scale_skeletons: scaled = 'scaled_' joint_set = 'no_hands_' ref_set = 'CoM_' cached_CD_file_name = os.path.join(self._root_path, (self._dataset_prefix + str(self._part_id) + '_' + scaled + joint_set + ref_set + 'CD_' + self._movement + '.npy')) cached_NM_file_name = os.path.join(self._root_path, (self._dataset_prefix + str(self._part_id) + '_' + scaled + joint_set + ref_set + 'NM_' + self._movement + '.npy')) cd_cache_file_exists = os.path.exists(cached_CD_file_name) nm_cache_file_exists = os.path.exists(cached_NM_file_name) if not cd_cache_file_exists or not nm_cache_file_exists or rebuild: cosine_distance = [] normalised_magnitude = [] for skel_idx in tqdm(range(np.shape(self.stacked_filtered_XYZ_values)[2])): skel = self.stacked_filtered_XYZ_values[:,:,skel_idx] cd_row = [] nm_row = [] col_names = [] for ref_joint in RefernceTorsoJoints_COM: #RefernceTorsoJoints_HEAD RefernceTorsoJoints_NECK RefernceTorsoJoints_COM for skel_joint in SkeletonJoints_no_hands: #SkeletonJoints cd = sm.cosine_distance_between_joints(skel[:, ref_joint.value], skel[:, skel_joint.value]) col_names.append(ref_joint.name + '_' + skel_joint.name) # cd_csv_rows.append({'ref_joint': ref_joint.name, # 'skel_joint': skel_joint.name, # 'cd': cd}) cd_row.append(cd) nm = sm.normalised_magnitude_between_joints(skel[:, skel_joint.value], skel[:, ref_joint.value]) # nm_csv_rows.append({'ref_joint': ref_joint.name, # 'skel_joint': skel_joint.name, # 'nm': nm}) nm_row.append(nm) cosine_distance.append(cd_row) normalised_magnitude.append(nm_row) self.cosine_distance = cosine_distance self.normalised_magnitude = normalised_magnitude #save np.save(cached_CD_file_name, self.cosine_distance) np.save(cached_NM_file_name, self.normalised_magnitude) # for col in SkeletonJoints_no_hands: # col_names.append(col.name) pd.DataFrame(self.cosine_distance, columns=col_names).to_csv(cached_CD_file_name.replace('.npy', '.csv')) pd.DataFrame(self.normalised_magnitude, columns=col_names).to_csv(cached_NM_file_name.replace('.npy', '.csv')) else: self.cosine_distance = np.load(cached_CD_file_name) print('loading:', cached_CD_file_name, '\n') self.normalised_magnitude = np.load(cached_NM_file_name) print('loading:', cached_NM_file_name, '\n') if save_pngs: str_part_id = str(self._part_id) plt.figure() plt.plot(kinect_recording.cosine_distance) plt.title('Cosign distance ' + scaled + ' ' + str_part_id) plt.savefig(os.path.join(self._root_path,('CD_' + scaled + ref_set + joint_set + str_part_id))) plt.close() plt.figure() plt.plot(kinect_recording.normalised_magnitude) plt.title('Normalised magnitude ' + scaled + ' ' + str_part_id) plt.savefig(os.path.join(self._root_path,('NM_' + scaled + ref_set + joint_set + str_part_id))) plt.close() img = pd.DataFrame(self.cosine_distance).values #min_max_scaler = preprocessing.MinMaxScaler() img_scaled = ((img - img.min()) * (1/(img.max() - img.min()) * 255)).astype('uint8') img_3ch = np.dstack([img_scaled, img_scaled, img_scaled]) img_3ch_to_save = pil_image.fromarray(img_3ch, mode='RGB') img_3ch_to_save.save(cached_NM_file_name.replace('.npy', '.png')) def normalise_skeleton(self, skel_frame): if len(self._ref_spine_base) == 0: self._ref_spine_base = skel_frame.iloc[SkeletonJoints.SPINEBASE.value][['X', 'Y', 'Z']].tolist() self._ref_skel_frame = skel_frame normalised_skel_frame = pd.DataFrame.copy(skel_frame,deep=False) # x = 0 # y = 1 # z = 2 for i, joint in enumerate(SkeletonJoints): #HierarchyForBodySegments joint_name = joint.name joint_number = joint.value normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('X')] = (skel_frame.iloc[joint_number]['X'] - self._ref_spine_base[X]) normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('Y')] = (skel_frame.iloc[joint_number]['Y'] - self._ref_spine_base[Y]) normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('Z')] = (skel_frame.iloc[joint_number]['Z'] - self._ref_spine_base[Z]) if self._scale_skeletons: skel_scale = self.get_scale_for_joint(skel_frame, joint_name) normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('X')] = (normalised_skel_frame.iloc[joint_number]['X'] * skel_scale) normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('Y')] = (normalised_skel_frame.iloc[joint_number]['Y'] * skel_scale) normalised_skel_frame.iloc[joint_number, normalised_skel_frame.columns.get_loc('Z')] = (normalised_skel_frame.iloc[joint_number]['Z'] * skel_scale) if joint_number == 24: break self._frame_count +=1 return normalised_skel_frame def _load_skel_file(self, skel_file_path): """ debug skelfile """ #print(skel_file_path) #replace with array of skeletons columns = ['Joint', 'Tracked', 'X', 'Y', 'Z', 'px_X', 'px_Y'] skel_frame = pd.read_csv(skel_file_path, delimiter=' ', header=None, nrows=25, names=columns, index_col=0) ''' Normalise ''' skel_frame = self.normalise_skeleton(skel_frame) #Add CoM tmp_CoM = self.calulate_CoM_position(skel_frame) tmp_CoM_row = {'Tracked':'Tracked', 'X':tmp_CoM[0], 'Y':tmp_CoM[1], 'Z':tmp_CoM[2], 'px_X':0, 'px_Y':0} df_CoM_row = pd.DataFrame(tmp_CoM_row, index=['CoM']) skel_frame = skel_frame.append(df_CoM_row) X = skel_frame['X'] Y = skel_frame['Y'] Z = skel_frame['Z'] raw_XYZ = np.stack([X.values, Y.values, Z.values]) return skel_frame, raw_XYZ def calulate_CoM_position(self, skel_frame): # _X = 2 # _Y = 3 # _Z = 4 spine_base = np.stack([skel_frame['X'][SkeletonJoints.SPINEMID.value], skel_frame['Y'][SkeletonJoints.SPINEMID.value], skel_frame['Z'][SkeletonJoints.SPINEMID.value]]) hip_left = np.stack([skel_frame['X'][SkeletonJoints.HIPLEFT.value], skel_frame['Y'][SkeletonJoints.HIPLEFT.value], skel_frame['Z'][SkeletonJoints.HIPLEFT.value]]) hip_right = np.stack([skel_frame['X'][SkeletonJoints.HIPRIGHT.value], skel_frame['Y'][SkeletonJoints.HIPRIGHT.value], skel_frame['Z'][SkeletonJoints.HIPRIGHT.value]]) x_mean = np.mean([spine_base[0],hip_left[0],hip_right[0]]) y_mean = np.mean([spine_base[1],hip_left[1],hip_right[1]]) z_mean = np.mean([spine_base[2],hip_left[2],hip_right[2]]) CoM = np.stack([x_mean,y_mean,z_mean]) #com = spine_base return CoM.tolist() def calulate_skeleton_angles_from_stacked_XYZ_values(self, stacked_filtered_XYZ_values): angles_list = [] #X = 0 #Y = 1 #Z = 2 tmp_foot_left_joint = np.stack([np.mean(stacked_filtered_XYZ_values[X][SkeletonJoints.FOOTLEFT.value]), np.mean(stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTLEFT.value]), np.mean(stacked_filtered_XYZ_values[Z][SkeletonJoints.FOOTLEFT.value])]) tmp_foot_left_joint = np.stack([np.mean(stacked_filtered_XYZ_values[X][SkeletonJoints.FOOTRIGHT.value]), np.mean(stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTRIGHT.value]), np.mean(stacked_filtered_XYZ_values[Z][SkeletonJoints.FOOTRIGHT.value])]) tmp_foot_mean_joint = sm.mean_twin_joint_pos(tmp_foot_left_joint, tmp_foot_left_joint) tmp_ground_plane = np.stack([np.mean(stacked_filtered_XYZ_values[X][SkeletonJoints.COM.value]), tmp_foot_mean_joint[1], np.mean(stacked_filtered_XYZ_values[Z][SkeletonJoints.COM.value])]) for i in np.ndindex(stacked_filtered_XYZ_values.shape[2]): tmp_shoulder_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.SHOULDERLEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.SHOULDERLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.SHOULDERLEFT.value][i]]) tmp_shoulder_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.SHOULDERRIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.SHOULDERRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.SHOULDERRIGHT.value][i]]) tmp_spine_shoulder_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.SPINESHOULDER.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.SPINESHOULDER.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.SPINESHOULDER.value][i]]) tmp_com_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.COM.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.COM.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.COM.value][i]]) tmp_elbow_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ELBOWLEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ELBOWLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ELBOWLEFT.value][i]]) tmp_elbow_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ELBOWRIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ELBOWRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ELBOWRIGHT.value][i]]) tmp_wrist_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.WRISTLEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.WRISTLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.WRISTLEFT.value][i]]) tmp_wrist_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.WRISTRIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.WRISTRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.WRISTRIGHT.value][i]]) tmp_hip_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.HIPLEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.HIPLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.HIPLEFT.value][i]]) tmp_hip_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.HIPRIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.HIPRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.HIPRIGHT.value][i]]) tmp_knee_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.KNEELEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.KNEELEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.KNEELEFT.value][i]]) tmp_knee_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.KNEERIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.KNEERIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.KNEERIGHT.value][i]]) tmp_ankle_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLELEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ANKLELEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLELEFT.value][i]]) tmp_ankle_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLERIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ANKLERIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLERIGHT.value][i]]) tmp_heel_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLELEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLELEFT.value][i]]) tmp_heel_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLERIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLERIGHT.value][i]]) tmp_heel_mean_joint = sm.mean_twin_joint_pos(tmp_heel_left_joint, tmp_heel_right_joint) tmp_ankle_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLELEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ANKLELEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLELEFT.value][i]]) tmp_ankle_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.ANKLERIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.ANKLERIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.ANKLERIGHT.value][i]]) tmp_foot_left_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.FOOTLEFT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTLEFT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.FOOTLEFT.value][i]]) tmp_foot_right_joint = np.stack([stacked_filtered_XYZ_values[X][SkeletonJoints.FOOTRIGHT.value][i], stacked_filtered_XYZ_values[Y][SkeletonJoints.FOOTRIGHT.value][i], stacked_filtered_XYZ_values[Z][SkeletonJoints.FOOTRIGHT.value][i]]) body_com_angle = sm.get_angle_between_three_joints(tmp_com_joint, tmp_heel_mean_joint, tmp_ground_plane) body_lean_angle = sm.get_angle_between_three_joints(tmp_spine_shoulder_joint, tmp_heel_mean_joint, tmp_ground_plane) knee_angle_left = sm.get_angle_between_three_joints(tmp_hip_left_joint, tmp_knee_left_joint, tmp_ankle_left_joint) knee_angle_right = sm.get_angle_between_three_joints(tmp_hip_right_joint, tmp_knee_right_joint, tmp_ankle_right_joint) hip_angle_left = sm.get_angle_between_three_joints(tmp_shoulder_left_joint, tmp_hip_left_joint, tmp_knee_left_joint) hip_angle_right = sm.get_angle_between_three_joints(tmp_shoulder_right_joint, tmp_hip_right_joint, tmp_knee_right_joint) elbow_angle_left = sm.get_angle_between_three_joints(tmp_shoulder_left_joint, tmp_elbow_left_joint, tmp_wrist_left_joint) elbow_angle_right = sm.get_angle_between_three_joints(tmp_shoulder_right_joint, tmp_elbow_right_joint, tmp_wrist_right_joint) armpit_angle_left = sm.get_angle_between_three_joints(tmp_hip_left_joint, tmp_shoulder_left_joint, tmp_elbow_left_joint) armpit_angle_right = sm.get_angle_between_three_joints(tmp_hip_right_joint, tmp_shoulder_right_joint, tmp_elbow_right_joint) ankle_angle_left = sm.get_angle_between_three_joints(tmp_foot_left_joint, tmp_heel_left_joint, tmp_knee_left_joint) ankle_angle_right = sm.get_angle_between_three_joints(tmp_foot_right_joint, tmp_heel_right_joint, tmp_knee_right_joint) ''' BODY_COM_ANGLE = 1 BODY_LEAN_ANGLE = 2 KNEELEFT_ANGLE = 3 KNEERIGHT_ANGLE = 4 HIPLEFT_ANGLE = 5 HIPRIGHT_ANGLE = 6 ELBOWLEFT_ANGLE = 7 ELBOWLEFT_ANGLE = 8 ARMPITLEFT_ANGLE = 9 ARMPITLEFT_ANGLE = 10 ANKLELEFT_ANGLE = 11 ANKLELRIGHT_ANGLE = 12 ''' angles_row = np.hstack([self._part_id, body_com_angle, body_lean_angle, knee_angle_left, knee_angle_right, hip_angle_left, hip_angle_right, elbow_angle_left, elbow_angle_right, armpit_angle_left, armpit_angle_right, ankle_angle_left, ankle_angle_right]) if len(self._labels) != 0: arr_labels = np.array(self._labels)[0] self.load_sway_metrics() angles_row = np.concatenate((angles_row, np.array(self.sway_metrics.iloc[0,10:]), arr_labels)) angles_list.append(angles_row) return np.array(angles_list) def calculate_walked_skel_angles(self): columns = [] for combination in WalkedSkelAnglesIn3s: #for combination in WalkedSkelAngles: columns = np.hstack([columns, (combination.name + '_SP')]) columns = np.hstack([columns, (combination.name + '_FP')]) columns = np.hstack([columns, (combination.name + '_TP')]) skel_walked_angels = [] X = 0 Y = 1 Z = 2 for i in range(self.stacked_filtered_XYZ_values.shape[2]): skel_angles_row = np.array([]) for combination in WalkedSkelAnglesIn3s: #for combination in WalkedSkelAngles: j1 = np.stack([1 * self.stacked_filtered_XYZ_values[X][combination.value[0]][i], 1 * self.stacked_filtered_XYZ_values[Y][combination.value[0]][i], 1 * self.stacked_filtered_XYZ_values[Z][combination.value[0]][i]]) j2 = np.stack([1 * self.stacked_filtered_XYZ_values[X][combination.value[1]][i], 1 * self.stacked_filtered_XYZ_values[Y][combination.value[1]][i], 1 * self.stacked_filtered_XYZ_values[Z][combination.value[1]][i]]) j3 = np.stack([1 * self.stacked_filtered_XYZ_values[X][combination.value[2]][i], 1 * self.stacked_filtered_XYZ_values[Y][combination.value[2]][i], 1 * self.stacked_filtered_XYZ_values[Z][combination.value[2]][i]]) euler_angle_3d = sm.get_3d_angle_between_three_joints(j1, j2, j3, deg=True) if len(skel_angles_row) == 0: skel_angles_row = euler_angle_3d else: skel_angles_row = np.hstack([skel_angles_row, euler_angle_3d]) skel_walked_angels.append(skel_angles_row) pd_skel_walked_angels = pd.DataFrame(skel_walked_angels, columns=columns) self.walked_skel_angles = pd_skel_walked_angels