|
a |
|
b/utils/ramps.py |
|
|
1 |
# Copyright (c) 2018, Curious AI Ltd. All rights reserved. |
|
|
2 |
# |
|
|
3 |
# This work is licensed under the Creative Commons Attribution-NonCommercial |
|
|
4 |
# 4.0 International License. To view a copy of this license, visit |
|
|
5 |
# http://creativecommons.org/licenses/by-nc/4.0/ or send a letter to |
|
|
6 |
# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. |
|
|
7 |
|
|
|
8 |
"""Functions for ramping hyperparameters up or down |
|
|
9 |
|
|
|
10 |
Each function takes the current training step or epoch, and the |
|
|
11 |
ramp length in the same format, and returns a multiplier between |
|
|
12 |
0 and 1. |
|
|
13 |
""" |
|
|
14 |
|
|
|
15 |
|
|
|
16 |
import numpy as np |
|
|
17 |
|
|
|
18 |
|
|
|
19 |
def sigmoid_rampup(current, rampup_length): |
|
|
20 |
"""Exponential rampup from https://arxiv.org/abs/1610.02242""" |
|
|
21 |
if rampup_length == 0: |
|
|
22 |
return 1.0 |
|
|
23 |
else: |
|
|
24 |
current = np.clip(current, 0.0, rampup_length) |
|
|
25 |
phase = 1.0 - current / rampup_length |
|
|
26 |
return float(np.exp(-5.0 * phase * phase)) |
|
|
27 |
|
|
|
28 |
|
|
|
29 |
def linear_rampup(current, rampup_length): |
|
|
30 |
"""Linear rampup""" |
|
|
31 |
assert current >= 0 and rampup_length >= 0 |
|
|
32 |
if current >= rampup_length: |
|
|
33 |
return 1.0 |
|
|
34 |
else: |
|
|
35 |
return current / rampup_length |
|
|
36 |
|
|
|
37 |
|
|
|
38 |
def cosine_rampdown(current, rampdown_length): |
|
|
39 |
"""Cosine rampdown from https://arxiv.org/abs/1608.03983""" |
|
|
40 |
assert 0 <= current <= rampdown_length |
|
|
41 |
return float(.5 * (np.cos(np.pi * current / rampdown_length) + 1)) |