Log In Sign Up
Models:
Philip Bordner
philipB/
Targeted-BEHRT
0
Downloads: 1
[9e1f38]: / pytorch_pretrained_bert / optimizer.py

Download this file

69 lines (57 with data), 3.2 kB 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
import pytorch_pretrained_bert as Bert



def VAEadam(params, config=None):

    if config is None:

        config = {

            'lr': 3e-5,

            'warmup_proportion': 0.1

        }

    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight', 'Eps']

    vae= ['VAE']

    no_decayfull = no_decay+vae

#     print( {'params': [n for n, p in params if not any(nd in n for nd in no_decayfull)], 'weight_decay': 0.01, 'lr': config['lr']},

#         {'params': [n for n, p in params if any(nd in n for nd in no_decay)], 'weight_decay': 0.0, 'lr': config['lr']},

#         {'params': [n for n, p in params if any(nd in n for nd in vae)], 'weight_decay': 0.0, 'lr':1e-3 }

# )

    optimizer_grouped_parameters = [

        {'params': [p for n, p in params if not any(nd in n for nd in no_decayfull)], 'weight_decay': 0.01, 'lr': config['lr']},

        {'params': [p for n, p in params if (any(nd in n for nd in no_decay) and 'VAE' not in n)], 'weight_decay': 0.0, 'lr': config['lr']},

        {'params': [p for n, p in params if any(nd in n for nd in vae)], 'weight_decay': 0.0, 'lr':1e-3 }



    ]



    optim = Bert.optimization.BertAdam(optimizer_grouped_parameters,

                                       warmup=config['warmup_proportion'])

    return optim





def adam(params, config=None):

    if config is None:

        config = {

            'lr': 3e-5,

            'warmup_proportion': 0.1

        }

    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight', 'Eps','VAE']

    optimizer_grouped_parameters = [

        {'params': [p for n, p in params if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},

        {'params': [p for n, p in params if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

    ]



    optim = Bert.optimization.BertAdam(optimizer_grouped_parameters,

                                       lr=config['lr'],

                                       warmup=config['warmup_proportion'])

    return optim



def GPadam(params, gpLR, config=None):

    if config is None:

        config = {

            'lr': 3e-5,

            'warmup_proportion': 0.1

        }

    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight','Eps']

    gp = ['GP']







    optimizer_grouped_parameters = [

        {'params': [p for n, p in params if not any(nd in n for nd in no_decay) and not any(nd in n for nd in gp)], 'weight_decay': 0.01 , 'lr': config['lr'], 'warmup_proportion': 0.1},

        {'params': [p for n, p in params if any(nd in n for nd in no_decay) and not any(nd in n for nd in gp)], 'weight_decay': 0.0, 'lr': config['lr'], 'warmup_proportion': 0.1},

        {'params': [p for n, p in params if any(nd in n for nd in gp)], 'lr': gpLR}

    ]



    print([

        {'params': [n for n, p in params if not any(nd in n for nd in no_decay) and not any(nd in n for nd in gp)], 'weight_decay': 0.01 , 'lr': config['lr'], 'warmup_proportion': 0.1},

        {'params': [n for n, p in params if any(nd in n for nd in no_decay) and not any(nd in n for nd in gp)], 'weight_decay': 0.0, 'lr': config['lr'], 'warmup_proportion': 0.1},

        {'params': [n for n, p in params if any(nd in n for nd in gp)], 'lr': gpLR}

    ])

    optim = Bert.optimization.BertAdam(optimizer_grouped_parameters)

    return optim