[4fa73e]: / pytorch / utils / config.py

Download this file

98 lines (76 with data), 3.6 kB

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import os
import logging
from logging import Formatter
from logging.handlers import RotatingFileHandler
import json
from easydict import EasyDict
from pprint import pprint
from utils.dirs import create_dirs
def setup_logging(log_dir):
log_file_format = "[%(levelname)s] - %(asctime)s - %(name)s - : %(message)s in %(pathname)s:%(lineno)d"
log_console_format = "[%(levelname)s]: %(message)s"
# Main logger
main_logger = logging.getLogger()
main_logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(Formatter(log_console_format))
exp_file_handler = RotatingFileHandler('{}exp_debug.log'.format(log_dir), maxBytes=10**6, backupCount=5)
exp_file_handler.setLevel(logging.DEBUG)
exp_file_handler.setFormatter(Formatter(log_file_format))
exp_errors_file_handler = RotatingFileHandler('{}exp_error.log'.format(log_dir), maxBytes=10**6, backupCount=5)
exp_errors_file_handler.setLevel(logging.WARNING)
exp_errors_file_handler.setFormatter(Formatter(log_file_format))
main_logger.addHandler(console_handler)
main_logger.addHandler(exp_file_handler)
main_logger.addHandler(exp_errors_file_handler)
def get_config_from_json(json_file):
"""
Get the config from a json file
:param json_file: the path of the config file
:return: config(namespace), config(dictionary)
"""
# parse the configurations from the config json file provided
with open(json_file, 'r') as config_file:
try:
config_dict = json.load(config_file)
# EasyDict allows to access dict values as attributes (works recursively).
config = EasyDict(config_dict)
return config, config_dict
except ValueError:
print("INVALID JSON file format.. Please provide a good json file")
exit(-1)
def process_config(json_file, args):
"""
Get the json file
Processing it with EasyDict to be accessible as attributes
then editing the path of the experiments folder
creating some important directories in the experiment folder
Then setup the logging in the whole program
Then return the config
:param json_file: the path of the config file
:return: config object(namespace)
"""
config, _ = get_config_from_json(json_file)
print(" THE Configuration of your experiment ..")
pprint(config)
# making sure that you have provided the exp_name.
try:
print(" *************************************** ")
print("The experiment name is {}".format(config.exp_name))
print(" *************************************** ")
except AttributeError:
print("ERROR!!..Please provide the exp_name in json file..")
exit(-1)
# create some important directories to be used for that experiment.
config.summary_dir = os.path.join("experiments", config.exp_name, "summaries/")
config.checkpoint_dir = os.path.join("experiments", config.exp_name, "checkpoints/")
config.out_dir = os.path.join("experiments", config.exp_name, "out/")
config.log_dir = os.path.join("experiments", config.exp_name, "logs/")
create_dirs([config.summary_dir, config.checkpoint_dir, config.out_dir, config.log_dir])
# setup logging in the project
setup_logging(config.log_dir)
logging.getLogger().info("Hi, This is root.")
logging.getLogger().info("After the configurations are successfully processed and dirs are created.")
logging.getLogger().info("The pipeline of the project will begin now.")
return config