Diff of /app/utils/perflog.py [000000] .. [d6904d]

Switch to unified view

a b/app/utils/perflog.py
1
import json
2
import time
3
4
from numpyencoder import NumpyEncoder
5
from omegaconf import OmegaConf
6
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, create_engine
7
from sqlalchemy.ext.declarative import declarative_base
8
from sqlalchemy.orm import Session, relationship, sessionmaker
9
10
Base = declarative_base()
11
12
13
class Perflog(Base):
14
    __tablename__ = "perflog"
15
    id = Column(Integer, primary_key=True, index=True)
16
    dataset = Column(String)
17
    task = Column(String)
18
    model_type = Column(String)
19
    model_name = Column(String)
20
    hidden_dim = Column(Integer)
21
    performance = Column(String)
22
    config = Column(String)
23
    record_time = Column(Integer)
24
25
26
def process_performance_raw_info(
27
    cfg,
28
    mae=None,
29
    mse=None,
30
    rmse=None,
31
    mape=None,
32
    acc=None,
33
    auroc=None,
34
    auprc=None,
35
    early_prediction_score=None,
36
    multitask_prediction_score=None,
37
    verbose=0,
38
):
39
    result = []
40
    if mae is not None:
41
        result.extend(
42
            [
43
                {"name": "mae", "mean": mae.mean(), "std": mae.std()},
44
                {"name": "mse", "mean": mse.mean(), "std": mse.std()},
45
                {"name": "rmse", "mean": rmse.mean(), "std": rmse.std()},
46
                {"name": "mape", "mean": mape.mean(), "std": mape.std()},
47
            ]
48
        )
49
    if acc is not None:
50
        result.extend(
51
            [
52
                {"name": "acc", "mean": acc.mean(), "std": acc.std()},
53
                {"name": "auroc", "mean": auroc.mean(), "std": auroc.std()},
54
                {"name": "auprc", "mean": auprc.mean(), "std": auprc.std()},
55
            ]
56
        )
57
    thresholds = cfg.thresholds
58
    if early_prediction_score is not None:
59
        for i in range(len(thresholds)):
60
            result.append(
61
                {
62
                    "name": "early_prediction_score",
63
                    "mean": early_prediction_score.mean(axis=0)[i],
64
                    "std": early_prediction_score.std(axis=0)[i],
65
                    "threshold": thresholds[i],
66
                }
67
            )
68
    if multitask_prediction_score is not None:
69
        for i in range(len(thresholds)):
70
            result.append(
71
                {
72
                    "name": "multitask_prediction_score",
73
                    "mean": multitask_prediction_score.mean(axis=0)[i],
74
                    "std": multitask_prediction_score.std(axis=0)[i],
75
                    "threshold": thresholds[i],
76
                }
77
            )
78
    if verbose == 1:
79
        print(result)
80
    return result
81
82
83
def create_perflog(db: Session, cfg, perf=None):
84
    hid_dim = 0
85
    if "hidden_dim" in cfg:
86
        hid_dim = cfg.hidden_dim
87
    db_perflog = Perflog(
88
        task=cfg.task,
89
        dataset=cfg.dataset,
90
        model_type=cfg.model_type,
91
        model_name=cfg.model,
92
        hidden_dim=hid_dim,
93
        performance=json.dumps(perf, cls=NumpyEncoder),
94
        config=OmegaConf.to_yaml(cfg),
95
        record_time=int(time.time()),
96
    )
97
    db.add(db_perflog)
98
    db.commit()
99
    db.refresh(db_perflog)
100
    return db_perflog
101
102
103
def process_and_upload_performance(
104
    cfg,
105
    mae=None,
106
    mse=None,
107
    rmse=None,
108
    mape=None,
109
    acc=None,
110
    auroc=None,
111
    auprc=None,
112
    early_prediction_score=None,
113
    multitask_prediction_score=None,
114
    verbose=0,
115
    upload=False,
116
):
117
    perf = process_performance_raw_info(
118
        cfg,
119
        mae=mae,
120
        mse=mse,
121
        rmse=rmse,
122
        mape=mape,
123
        acc=acc,
124
        auroc=auroc,
125
        auprc=auprc,
126
        early_prediction_score=early_prediction_score,
127
        multitask_prediction_score=multitask_prediction_score,
128
        verbose=verbose,
129
    )
130
    if upload:
131
        db_cfg = OmegaConf.load("configs/_base_/db.yaml")
132
133
        engine, username, password, host, port, database = (
134
            db_cfg.engine,
135
            db_cfg.username,
136
            db_cfg.password,
137
            db_cfg.host,
138
            db_cfg.port,
139
            db_cfg.database,
140
        )
141
        SQLALCHEMY_DATABASE_URL = (
142
            f"{engine}://{username}:{password}@{host}:{port}/{database}"
143
        )
144
        engine = create_engine(SQLALCHEMY_DATABASE_URL)
145
        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
146
147
        db = SessionLocal()
148
        create_perflog(db=db, cfg=cfg, perf=perf)
149
        db.close()