|
a |
|
b/submission/baselines/bench/benchmarks.py |
|
|
1 |
import re |
|
|
2 |
import os.path as osp |
|
|
3 |
import os |
|
|
4 |
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
|
|
5 |
|
|
|
6 |
_atari7 = ['BeamRider', 'Breakout', 'Enduro', 'Pong', 'Qbert', 'Seaquest', 'SpaceInvaders'] |
|
|
7 |
_atariexpl7 = ['Freeway', 'Gravitar', 'MontezumaRevenge', 'Pitfall', 'PrivateEye', 'Solaris', 'Venture'] |
|
|
8 |
|
|
|
9 |
_BENCHMARKS = [] |
|
|
10 |
|
|
|
11 |
remove_version_re = re.compile(r'-v\d+$') |
|
|
12 |
|
|
|
13 |
|
|
|
14 |
def register_benchmark(benchmark): |
|
|
15 |
for b in _BENCHMARKS: |
|
|
16 |
if b['name'] == benchmark['name']: |
|
|
17 |
raise ValueError('Benchmark with name %s already registered!' % b['name']) |
|
|
18 |
|
|
|
19 |
# automatically add a description if it is not present |
|
|
20 |
if 'tasks' in benchmark: |
|
|
21 |
for t in benchmark['tasks']: |
|
|
22 |
if 'desc' not in t: |
|
|
23 |
t['desc'] = remove_version_re.sub('', t['env_id']) |
|
|
24 |
_BENCHMARKS.append(benchmark) |
|
|
25 |
|
|
|
26 |
|
|
|
27 |
def list_benchmarks(): |
|
|
28 |
return [b['name'] for b in _BENCHMARKS] |
|
|
29 |
|
|
|
30 |
|
|
|
31 |
def get_benchmark(benchmark_name): |
|
|
32 |
for b in _BENCHMARKS: |
|
|
33 |
if b['name'] == benchmark_name: |
|
|
34 |
return b |
|
|
35 |
raise ValueError('%s not found! Known benchmarks: %s' % (benchmark_name, list_benchmarks())) |
|
|
36 |
|
|
|
37 |
|
|
|
38 |
def get_task(benchmark, env_id): |
|
|
39 |
"""Get a task by env_id. Return None if the benchmark doesn't have the env""" |
|
|
40 |
return next(filter(lambda task: task['env_id'] == env_id, benchmark['tasks']), None) |
|
|
41 |
|
|
|
42 |
|
|
|
43 |
def find_task_for_env_id_in_any_benchmark(env_id): |
|
|
44 |
for bm in _BENCHMARKS: |
|
|
45 |
for task in bm["tasks"]: |
|
|
46 |
if task["env_id"] == env_id: |
|
|
47 |
return bm, task |
|
|
48 |
return None, None |
|
|
49 |
|
|
|
50 |
|
|
|
51 |
_ATARI_SUFFIX = 'NoFrameskip-v4' |
|
|
52 |
|
|
|
53 |
register_benchmark({ |
|
|
54 |
'name': 'Atari50M', |
|
|
55 |
'description': '7 Atari games from Mnih et al. (2013), with pixel observations, 50M timesteps', |
|
|
56 |
'tasks': [{'desc': _game, 'env_id': _game + _ATARI_SUFFIX, 'trials': 2, 'num_timesteps': int(50e6)} for _game in _atari7] |
|
|
57 |
}) |
|
|
58 |
|
|
|
59 |
register_benchmark({ |
|
|
60 |
'name': 'Atari10M', |
|
|
61 |
'description': '7 Atari games from Mnih et al. (2013), with pixel observations, 10M timesteps', |
|
|
62 |
'tasks': [{'desc': _game, 'env_id': _game + _ATARI_SUFFIX, 'trials': 2, 'num_timesteps': int(10e6)} for _game in _atari7] |
|
|
63 |
}) |
|
|
64 |
|
|
|
65 |
register_benchmark({ |
|
|
66 |
'name': 'Atari1Hr', |
|
|
67 |
'description': '7 Atari games from Mnih et al. (2013), with pixel observations, 1 hour of walltime', |
|
|
68 |
'tasks': [{'desc': _game, 'env_id': _game + _ATARI_SUFFIX, 'trials': 2, 'num_seconds': 60 * 60} for _game in _atari7] |
|
|
69 |
}) |
|
|
70 |
|
|
|
71 |
register_benchmark({ |
|
|
72 |
'name': 'AtariExploration10M', |
|
|
73 |
'description': '7 Atari games emphasizing exploration, with pixel observations, 10M timesteps', |
|
|
74 |
'tasks': [{'desc': _game, 'env_id': _game + _ATARI_SUFFIX, 'trials': 2, 'num_timesteps': int(10e6)} for _game in _atariexpl7] |
|
|
75 |
}) |
|
|
76 |
|
|
|
77 |
|
|
|
78 |
# MuJoCo |
|
|
79 |
|
|
|
80 |
_mujocosmall = [ |
|
|
81 |
'InvertedDoublePendulum-v2', 'InvertedPendulum-v2', |
|
|
82 |
'HalfCheetah-v2', 'Hopper-v2', 'Walker2d-v2', |
|
|
83 |
'Reacher-v2', 'Swimmer-v2'] |
|
|
84 |
register_benchmark({ |
|
|
85 |
'name': 'Mujoco1M', |
|
|
86 |
'description': 'Some small 2D MuJoCo tasks, run for 1M timesteps', |
|
|
87 |
'tasks': [{'env_id': _envid, 'trials': 3, 'num_timesteps': int(1e6)} for _envid in _mujocosmall] |
|
|
88 |
}) |
|
|
89 |
register_benchmark({ |
|
|
90 |
'name': 'MujocoWalkers', |
|
|
91 |
'description': 'MuJoCo forward walkers, run for 8M, humanoid 100M', |
|
|
92 |
'tasks': [ |
|
|
93 |
{'env_id': "Hopper-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
94 |
{'env_id': "Walker2d-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
95 |
{'env_id': "Humanoid-v1", 'trials': 4, 'num_timesteps': 100 * 1000000}, |
|
|
96 |
] |
|
|
97 |
}) |
|
|
98 |
|
|
|
99 |
# Roboschool |
|
|
100 |
|
|
|
101 |
register_benchmark({ |
|
|
102 |
'name': 'Roboschool8M', |
|
|
103 |
'description': 'Small 2D tasks, up to 30 minutes to complete on 8 cores', |
|
|
104 |
'tasks': [ |
|
|
105 |
{'env_id': "RoboschoolReacher-v1", 'trials': 4, 'num_timesteps': 2 * 1000000}, |
|
|
106 |
{'env_id': "RoboschoolAnt-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
107 |
{'env_id': "RoboschoolHalfCheetah-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
108 |
{'env_id': "RoboschoolHopper-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
109 |
{'env_id': "RoboschoolWalker2d-v1", 'trials': 4, 'num_timesteps': 8 * 1000000}, |
|
|
110 |
] |
|
|
111 |
}) |
|
|
112 |
register_benchmark({ |
|
|
113 |
'name': 'RoboschoolHarder', |
|
|
114 |
'description': 'Test your might!!! Up to 12 hours on 32 cores', |
|
|
115 |
'tasks': [ |
|
|
116 |
{'env_id': "RoboschoolHumanoid-v1", 'trials': 4, 'num_timesteps': 100 * 1000000}, |
|
|
117 |
{'env_id': "RoboschoolHumanoidFlagrun-v1", 'trials': 4, 'num_timesteps': 200 * 1000000}, |
|
|
118 |
{'env_id': "RoboschoolHumanoidFlagrunHarder-v1", 'trials': 4, 'num_timesteps': 400 * 1000000}, |
|
|
119 |
] |
|
|
120 |
}) |
|
|
121 |
|
|
|
122 |
# Other |
|
|
123 |
|
|
|
124 |
_atari50 = [ # actually 47 |
|
|
125 |
'Alien', 'Amidar', 'Assault', 'Asterix', 'Asteroids', |
|
|
126 |
'Atlantis', 'BankHeist', 'BattleZone', 'BeamRider', 'Bowling', |
|
|
127 |
'Breakout', 'Centipede', 'ChopperCommand', 'CrazyClimber', |
|
|
128 |
'DemonAttack', 'DoubleDunk', 'Enduro', 'FishingDerby', 'Freeway', |
|
|
129 |
'Frostbite', 'Gopher', 'Gravitar', 'IceHockey', 'Jamesbond', |
|
|
130 |
'Kangaroo', 'Krull', 'KungFuMaster', 'MontezumaRevenge', 'MsPacman', |
|
|
131 |
'NameThisGame', 'Pitfall', 'Pong', 'PrivateEye', 'Qbert', |
|
|
132 |
'RoadRunner', 'Robotank', 'Seaquest', 'SpaceInvaders', 'StarGunner', |
|
|
133 |
'Tennis', 'TimePilot', 'Tutankham', 'UpNDown', 'Venture', |
|
|
134 |
'VideoPinball', 'WizardOfWor', 'Zaxxon', |
|
|
135 |
] |
|
|
136 |
|
|
|
137 |
register_benchmark({ |
|
|
138 |
'name': 'Atari50_10M', |
|
|
139 |
'description': '47 Atari games from Mnih et al. (2013), with pixel observations, 10M timesteps', |
|
|
140 |
'tasks': [{'desc': _game, 'env_id': _game + _ATARI_SUFFIX, 'trials': 2, 'num_timesteps': int(10e6)} for _game in _atari50] |
|
|
141 |
}) |
|
|
142 |
|
|
|
143 |
# HER DDPG |
|
|
144 |
|
|
|
145 |
register_benchmark({ |
|
|
146 |
'name': 'HerDdpg', |
|
|
147 |
'description': 'Smoke-test only benchmark of HER', |
|
|
148 |
'tasks': [{'trials': 1, 'env_id': 'FetchReach-v1'}] |
|
|
149 |
}) |
|
|
150 |
|