from rdpg import *

import opensim as osim

from osim.http.client import Client

from osim.env import *

from history import History

ENV_NAME = 'learning_to_run'

PATH = 'models/'

EPISODES = 100000

TEST = 5

def main():

    env = RunEnv(visualize=False)

    env.reset(difficulty = 0)

    agent = RDPG(env)

    returns = []

    rewards = []

    for episode in xrange(EPISODES):

        state = env.reset(difficulty = 0)

        reward_episode = []

        print "episode:",episode

        #Initializing empty history

        history = History(state)

        # Train

        for step in xrange(env.spec.timestep_limit):

            action = agent.noise_action(history)

            next_state,reward,done,_ = env.step(action)

            # appending to history

            history.append(next_state,action,reward)

            reward_episode.append(reward)

            if done:

                break

        # storing the history into replay buffer and if the number of histories sequence is above the threshod, start training

        agent.perceive(history)

        # Testing:

        #if episode % 1 == 0:

        # if episode % 1000 == 0 and episode > 50:

        #     agent.save_model(PATH, episode)

        #     total_return = 0

        #     ave_reward = 0

        #     for i in xrange(TEST):

        #         state = env.reset()

        #         reward_per_step = 0

        #         for j in xrange(env.spec.timestep_limit):

        #             action = agent.action(state) # direct action for test

        #             state,reward,done,_ = env.step(action)

        #             total_return += reward

        #             if done:

        #                 break

        #             reward_per_step += (reward - reward_per_step)/(j+1)

        #         ave_reward += reward_per_step

        #     ave_return = total_return/TEST

        #     ave_reward = ave_reward/TEST

        #     returns.append(ave_return)

        #     rewards.append(ave_reward)

        #     print 'episode: ',episode,'Evaluation Average Return:',ave_return, '  Evaluation Average Reward: ', ave_reward

if __name__ == '__main__':

    main()