|
a |
|
b/dosma/utils/logger.py |
|
|
1 |
"""Logging utility. |
|
|
2 |
""" |
|
|
3 |
|
|
|
4 |
import atexit |
|
|
5 |
import functools |
|
|
6 |
import logging |
|
|
7 |
import os |
|
|
8 |
import sys |
|
|
9 |
from time import localtime, strftime |
|
|
10 |
from typing import Union |
|
|
11 |
|
|
|
12 |
from termcolor import colored |
|
|
13 |
|
|
|
14 |
from dosma.utils import env |
|
|
15 |
|
|
|
16 |
__all__ = ["setup_logger"] |
|
|
17 |
|
|
|
18 |
|
|
|
19 |
class _ColorfulFormatter(logging.Formatter): |
|
|
20 |
""" |
|
|
21 |
This class is adapted from Facebook's detectron2: |
|
|
22 |
https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/logger.py |
|
|
23 |
""" |
|
|
24 |
|
|
|
25 |
def __init__(self, *args, **kwargs): |
|
|
26 |
self._root_name = kwargs.pop("root_name") + "." |
|
|
27 |
self._abbrev_name = kwargs.pop("abbrev_name", "") |
|
|
28 |
if len(self._abbrev_name): |
|
|
29 |
self._abbrev_name = self._abbrev_name + "." |
|
|
30 |
super(_ColorfulFormatter, self).__init__(*args, **kwargs) |
|
|
31 |
|
|
|
32 |
def formatMessage(self, record): |
|
|
33 |
record.name = record.name.replace(self._root_name, self._abbrev_name) |
|
|
34 |
log = super(_ColorfulFormatter, self).formatMessage(record) |
|
|
35 |
if record.levelno == logging.WARNING: |
|
|
36 |
prefix = colored("WARNING", "red", attrs=["blink"]) |
|
|
37 |
elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: |
|
|
38 |
prefix = colored("ERROR", "red", attrs=["blink", "underline"]) |
|
|
39 |
else: |
|
|
40 |
return log |
|
|
41 |
return prefix + " " + log |
|
|
42 |
|
|
|
43 |
|
|
|
44 |
@functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers |
|
|
45 |
def setup_logger( |
|
|
46 |
output: Union[str, bool] = True, |
|
|
47 |
color=True, |
|
|
48 |
name="dosma", |
|
|
49 |
abbrev_name=None, |
|
|
50 |
stream_lvl=None, |
|
|
51 |
overwrite_handlers: bool = False, |
|
|
52 |
): |
|
|
53 |
"""Initialize the dosma logger. |
|
|
54 |
|
|
|
55 |
Args: |
|
|
56 |
output (str | bool): A file name or a directory to save log or a boolean. |
|
|
57 |
If ``True``, logs will save to the default dosma log location |
|
|
58 |
(:func:`dosma.utils.env.log_file_path`). |
|
|
59 |
If ``None`` or ``False``, logs will not be written to a file. This is not recommended. |
|
|
60 |
If a string and ends with ".txt" or ".log", assumed to be a file name. |
|
|
61 |
Otherwise, logs will be saved to `output/log.txt`. |
|
|
62 |
color (bool): If ``True``, logs printed to terminal (stdout) will be in color. |
|
|
63 |
name (str): The root module name of this logger. |
|
|
64 |
abbrev_name (str): An abbreviation of the module, to avoid long names in logs. |
|
|
65 |
Set to "" to not log the root module in logs. |
|
|
66 |
By default, will abbreviate "dosma" to "dm" and leave other |
|
|
67 |
modules unchanged. |
|
|
68 |
stream_lvl (int): The level for logging to console. Defaults to ``logging.DEBUG`` |
|
|
69 |
if :func:`dosma.utils.env.debug()` is ``True``, else defaults to ``logging.INFO``. |
|
|
70 |
overwrite_handlers (bool): It ``True`` and logger with name ``name`` has logging handlers, |
|
|
71 |
these handlers will be removed before adding the new handlers. This is useful |
|
|
72 |
when to avoid having too many handlers for a logger. |
|
|
73 |
|
|
|
74 |
Returns: |
|
|
75 |
logging.Logger: A logger. |
|
|
76 |
|
|
|
77 |
Note: |
|
|
78 |
This method removes existing handlers from the logger. |
|
|
79 |
|
|
|
80 |
Examples: |
|
|
81 |
>>> setup_logger() # how initializing logger is done most of the time |
|
|
82 |
>>> setup_logger("/path/to/save/dosma.log") # save log to particular file |
|
|
83 |
>>> setup_logger( |
|
|
84 |
... stream_lvl=logging.WARNING, |
|
|
85 |
... overwrite_handlers=True) # only prints warnings to console |
|
|
86 |
""" |
|
|
87 |
if stream_lvl is None: |
|
|
88 |
stream_lvl = logging.DEBUG if env.debug() else logging.INFO |
|
|
89 |
|
|
|
90 |
logger = logging.getLogger(name) |
|
|
91 |
logger.setLevel(logging.DEBUG) |
|
|
92 |
logger.propagate = False |
|
|
93 |
|
|
|
94 |
# Clear handlers if they exist. |
|
|
95 |
is_new_logger = not logger.hasHandlers() |
|
|
96 |
if not is_new_logger and overwrite_handlers: |
|
|
97 |
logger.handlers.clear() |
|
|
98 |
|
|
|
99 |
if abbrev_name is None: |
|
|
100 |
abbrev_name = "dm" if name == "dosma" else name |
|
|
101 |
|
|
|
102 |
plain_formatter = logging.Formatter( |
|
|
103 |
"[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%m/%d %H:%M:%S" |
|
|
104 |
) |
|
|
105 |
|
|
|
106 |
ch = logging.StreamHandler(stream=sys.stdout) |
|
|
107 |
ch.setLevel(stream_lvl) |
|
|
108 |
if color: |
|
|
109 |
formatter = _ColorfulFormatter( |
|
|
110 |
colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", |
|
|
111 |
datefmt="%m/%d %H:%M:%S", |
|
|
112 |
root_name=name, |
|
|
113 |
abbrev_name=str(abbrev_name), |
|
|
114 |
) |
|
|
115 |
else: |
|
|
116 |
formatter = plain_formatter |
|
|
117 |
ch.setFormatter(formatter) |
|
|
118 |
logger.addHandler(ch) |
|
|
119 |
|
|
|
120 |
if output is not None and output is not False: |
|
|
121 |
if output is True: |
|
|
122 |
output = env.log_file_path() |
|
|
123 |
|
|
|
124 |
if output.endswith(".txt") or output.endswith(".log"): |
|
|
125 |
filename = output |
|
|
126 |
else: |
|
|
127 |
filename = os.path.join(output, "dosma.log") |
|
|
128 |
os.makedirs(os.path.dirname(filename), exist_ok=True) |
|
|
129 |
|
|
|
130 |
fh = logging.StreamHandler(_cached_log_stream(filename)) |
|
|
131 |
fh.setLevel(logging.DEBUG) |
|
|
132 |
fh.setFormatter(plain_formatter) |
|
|
133 |
logger.addHandler(fh) |
|
|
134 |
|
|
|
135 |
if is_new_logger and name == "dosma": |
|
|
136 |
logger.debug("\n" * 4) |
|
|
137 |
logger.debug("==" * 40) |
|
|
138 |
logger.debug("Timestamp: %s" % strftime("%Y-%m-%d %H:%M:%S", localtime())) |
|
|
139 |
logger.debug("\n\n") |
|
|
140 |
|
|
|
141 |
return logger |
|
|
142 |
|
|
|
143 |
|
|
|
144 |
# cache the opened file object, so that different calls to `setup_logger` |
|
|
145 |
# with the same file name can safely write to the same file. |
|
|
146 |
@functools.lru_cache(maxsize=None) |
|
|
147 |
def _cached_log_stream(filename): |
|
|
148 |
# use 1K buffer if writing to cloud storage |
|
|
149 |
io = open(filename, "a", buffering=1024 if "://" in filename else -1) |
|
|
150 |
atexit.register(io.close) |
|
|
151 |
return io |