import os
import time
import matplotlib
matplotlib.use('Agg')
from models import *
from util import *
from data_management import *
import numpy as np
from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping
from keras.models import load_model
import h5py
import glob
from sklearn.metrics import average_precision_score
import sys
sys.path.insert(0,'./animation')
#from plot_video_animation_3D import *
import pandas as pd
import matplotlib.pyplot as plt
from img_exp import ImgExp
root_drive = 'N:/FallDetection/Fall-Data/'
if not os.path.isdir(root_drive):
print('Using Sharcnet equivalent of root_drive')
root_drive = '/home/jjniatsl/project/jjniatsl/Fall-Data/'
root_drive = './'
class SeqExp(ImgExp):
'''
A autoencoder experiment based on sequence of images
Inherits get_thresh, save exp, and variable initialization
Attributes:
int win_len: window length, or the number of contigous frames forming a sample
'''
def __init__(self, model = None, model_name = None,
misc_save_info = None, batch_size = 32, model_type = None, \
callbacks_list = None, pre_load = None, initial_epoch = 0, epochs = 1, \
dset = 'Thermal', win_len = 8, hor_flip = False, img_width = 64, img_height = 64):
ImgExp.__init__(self, model = model, img_width = img_width,\
img_height = img_height, model_name = model_name\
, batch_size = batch_size, \
model_type = model_type, \
pre_load = pre_load, initial_epoch = initial_epoch,\
epochs = epochs, hor_flip = hor_flip,\
dset = dset)
self.win_len = win_len
def set_train_data(self, raw = False, mmap_mode = None):#TODO init windows from h5py if no npData found
'''
loads or initazlzes windowed train data, and sets self.train_data accordingly
'''
if self.dset == 'Thermal': #TODO rename npdata in accordance with new initialzer
to_load = root_drive + '/npData/train_data_NonFalls_proc_windowed_by_vid-win_{}.npy'.format(self.win_len)
else:
to_load = root_drive + '/npData/{}/ADL_data-proc-win_{}.npy'.format(self.dset, self.win_len)
if os.path.isfile(to_load):
print('npData found, loading..')
self.train_data = np.load(to_load, mmap_mode = mmap_mode)
else:
print('npData not found, initializing..')
self.train_data = init_windowed_arr(dset = self.dset, ADL_only = True, win_len = self.win_len,
img_width = self.img_width, img_height = self.img_height)
if self.hor_flip == True:
to_load_flip = './npData/hor_flip-by_window/{}'.format(os.path.basename(to_load))
data_flip = self.init_flipped_by_win(to_load_flip)
self.train_data = np.concatenate((self.train_data, data_flip), axis = 0)
# return self.train_data
def train(self, sample_weight = None):
"""
trains a sequential autoencoder on windowed data. That is, sequeneces of contigous frames
are reconstucted.
"""
model_name = self.model_name
base = './Checkpoints/{}'.format(self.dset)
if not os.path.isdir(base):
os.mkdir(base)
checkpointer = ModelCheckpoint( filepath = base + '/' + model_name + '-' + \
'{epoch:03d}-{loss:.3f}.hdf5', period = 100, verbose =1)
timestamp = time.time()
print('./Checkpoints/' + model_name + '-' + '.{epoch:03d}-{loss:.3f}.hdf5')
csv_logger = CSVLogger('./Logs/' + model_name + 'training-' + \
str(timestamp) + '.log')
#callbacks_list = [checkpointer, early_stopper, csv_logger]
callbacks_list = [csv_logger, checkpointer]
self.model.fit(self.train_data, self.train_data, epochs = self.epochs, batch_size = self.batch_size,\
verbose = 2, callbacks = callbacks_list, sample_weight = sample_weight)
self.save_exp()
def init_flipped_by_win(self, to_load_flip):
if os.path.isfile(to_load_flip):
data_flip = np.load(to_load_flip)
data_flip = data_flip.reshape(len(data_flip), self.train_data.shape[1], self.train_data.shape[2],\
self.train_data.shape[3],1)
return data_flip
else:
print('creating flipped by window data..')
data_flip = flip_windowed_arr(self.train_data)
return data_flip
def get_MSE(self, test_data, agg_type = 'x_std'):
'''
MSE for sequential data (video). Uses data chunking with memap for SDU-Filled. Assumes windowed
Params:
ndarray test_data: data used to test model (reconstrcut). Of
shape (samples, window length, img_width, img_height)
agg_type: how to aggregate windowde scores
Returns:
ndarray: Mean squared error between test_data windows and reconstructed windows, aggregated.
This gives (samples,) shape
'''
import time
img_width, img_height, win_len, model, stride = self.img_width, self.img_height, self.win_len, self.model, 1
print('test_data.shape', test_data.shape)
if test_data.shape[1] != win_len: #Not windowed
test_data = test_data.reshape(len(test_data), img_width, img_height, 1)
test_data = create_windowed_arr(test_data, stride, win_len)
start_time = time.time()
recons_seq = model.predict(test_data) #(samples-win_len+1, win_len, wd,ht,1)
print(recons_seq.shape)
elapsed_time = time.time() - start_time
print('elapsed time for num frames', elapsed_time, len(test_data))
recons_seq = recons_seq.reshape(len(recons_seq),win_len, img_height*img_width)#(samples-win_len+1, 5, wd*ht)
test_data = test_data.reshape(len(test_data),win_len, img_height*img_width)#(samples-win_len+1, 5, wd*ht)
RE = np.mean(np.power(test_data-recons_seq, 2), axis = 2)# (samples-win_len+1,win_len)
RE = agg_window(RE, agg_type)
return RE
def get_MSE_all_agg(self, test_data):
"""
Gets MSE for all aggregate types 'x_std', 'x_mean', 'in_std', 'in_mean'.
Params:
ndarray test_data: data used to test model (reconstrcut). Of
shape (samples, window length, img_width, img_height)
Returns:
dictionary with keys 'x_std', 'x_mean', 'in_std', 'in_mean', and values
ndarrays of shape (samples,)
"""
img_width, img_height, win_len, model = self.img_width, self.img_height, self.win_len, self.model
recons_seq = model.predict(test_data) #(samples-win_len+1, win_len, wd,ht,1)
recons_seq_or = recons_seq
recons_seq = recons_seq.reshape(len(recons_seq),win_len, img_height*img_width)#(samples-win_len+1, 5, wd*ht)
test_data = test_data.reshape(len(test_data),win_len, img_height*img_width)#(samples-win_len+1, 5, wd*ht)
RE = np.mean(np.power(test_data-recons_seq, 2), axis = 2)# (samples-win_len+1,win_len)
RE_dict = {}
agg_type_list = ['x_std', 'x_mean', 'in_std', 'in_mean']
for agg_type in agg_type_list:
RE_dict[agg_type] = agg_window(RE, agg_type)
return RE_dict, recons_seq_or
def test(self, animate = False):
'''
Gets AUC ROC/PR for all videos, using various (20) scoring schemes. Saves scores to
'./AEComparisons/all_scores/self.dset/self.model_name.csv'
Assumes self.model has been initialized
'''
dset, to_load, img_width, img_height = self.dset, self.pre_load, self.img_width, self.img_height
stride = 1
win_len = self.win_len
model = self.model #TODO this can go in constructor
model_name = os.path.basename(to_load).split('.')[0]
print(model_name)
print(model.summary())
aucs = []
std_total = []
mean_total = []
labels_total_l = []
vid_index = 0 #vid index TODO rename
vid_dir_keys_Fall = generate_vid_keys('Fall', dset = dset)
num_vids = len(vid_dir_keys_Fall)
print('num_vids', num_vids)
ROC_mat = np.ones((num_vids, 2*win_len + 2)) # 20 scores-Xstd,Xmean,tols std..,tols mean..
PR_mat = np.ones((num_vids, 2*win_len + 2))
path = root_drive + 'H5Data/{}/Data_set-{}-imgdim{}x{}.h5'.format(dset, dset, img_width, img_height)
if not os.path.isfile(path):
print('initializing h5py..')
init_videos(img_width = img_width, img_height = img_height, \
raw = False, dset = dset)
with h5py.File(path, 'r') as hf:
data_dict = hf['{}/Processed/Split_by_video'.format(dset)]
for Fall_name in vid_dir_keys_Fall:
print(Fall_name)
vid_total = data_dict[Fall_name]['Data'][:]
labels_total = data_dict[Fall_name]['Labels'][:]
display_name = Fall_name
test_labels = labels_total
test_data = vid_total.reshape(len(vid_total), img_width, img_height, 1)
test_data_windowed = create_windowed_arr(test_data, stride, win_len)
RE_dict, recons_seq = self.get_MSE_all_agg(test_data_windowed) #Return dict with value for each score style
in_mean = RE_dict['in_mean']
in_std = RE_dict['in_std']
x_std = RE_dict['x_std']
x_mean = RE_dict['x_mean']
std_total.append(x_std)
mean_total.append(x_mean)
labels_total_l.append(labels_total)
inwin_labels = labels_total[win_len-1:]
auc_x_std, conf_mat, g_mean, ap_x_std = get_output(labels = test_labels,\
predictions = x_std, data_option = 'NA', to_plot = False)
auc_x_mean, conf_mat, g_mean, ap_x_mean = get_output(labels = test_labels,\
predictions = x_mean, data_option = 'NA', to_plot = False)
ROC_mat[vid_index,0] = auc_x_std
ROC_mat[vid_index,1] = auc_x_mean
tol_mat, tol_keys = gather_auc_avg_per_tol(in_mean, in_std, labels = test_labels, win_len = win_len)
AUROC_tol = tol_mat[0]
AUPR_tol = tol_mat[1]
num_scores_tol = tol_mat.shape[1]
for k in range(num_scores_tol):
j = k+2 #start at 2, first two were for X_std and X_mean
ROC_mat[vid_index,j] = AUROC_tol[k]
PR_mat[vid_index,j] = AUPR_tol[k]
PR_mat[vid_index,0] = ap_x_std
PR_mat[vid_index,1] = ap_x_mean
vid_index += 1
if animate == True:
ani_dir = './Animation/{}/'.format(dset)
ani_dir = ani_dir + '/{}'.format(model_name)
if not os.path.isdir(ani_dir):
os.makedirs(ani_dir)
print('saving animation to {}'.format(ani_dir))
animate_fall_detect_Spresent(testfall = test_data, recons = recons_seq[:,int(np.floor(win_len/2)),:], \
scores = x_mean, to_save = ani_dir + '/{}.mp4'.format(Fall_name))
# break
print('ROC_mat.shape', ROC_mat.shape)
AUROC_avg = np.mean(ROC_mat, axis = 0)
AUROC_std = np.std(ROC_mat, axis = 0)
AUROC_avg_std = join_mean_std(AUROC_avg, AUROC_std)
# print(AUROC_std)
AUPR_avg = np.mean(PR_mat, axis = 0)
AUPR_std = np.std(PR_mat, axis = 0)
AUPR_avg_std = join_mean_std(AUPR_avg, AUPR_std)
total = np.vstack((AUROC_avg_std, AUPR_avg_std))
print(tol_keys)
df = pd.DataFrame(data = total, index = ['AUROC','AUPR'], columns = ['X-STD','X-Mean'] + tol_keys)
print(df)
base = './AEComparisons/all_scores/{}/'.format(self.dset)
if not os.path.isdir(base):
os.makedirs(base)
save_path = './AEComparisons/all_scores/{}/{}.csv'.format(dset, model_name)
print(save_path)
df.to_csv(save_path)
Datasets
Models
