Diff of /.dev/check_urls.py [000000] .. [4e96d3]

Switch to unified view

a b/.dev/check_urls.py
1
# Copyright (c) OpenMMLab. All rights reserved.
2
import logging
3
import os
4
from argparse import ArgumentParser
5
6
import requests
7
import yaml as yml
8
9
from mmseg.utils import get_root_logger
10
11
12
def check_url(url):
13
    """Check url response status.
14
15
    Args:
16
        url (str): url needed to check.
17
18
    Returns:
19
        int, bool: status code and check flag.
20
    """
21
    flag = True
22
    r = requests.head(url)
23
    status_code = r.status_code
24
    if status_code == 403 or status_code == 404:
25
        flag = False
26
27
    return status_code, flag
28
29
30
def parse_args():
31
    parser = ArgumentParser('url valid check.')
32
    parser.add_argument(
33
        '-m',
34
        '--model-name',
35
        type=str,
36
        help='Select the model needed to check')
37
38
    args = parser.parse_args()
39
    return args
40
41
42
def main():
43
    args = parse_args()
44
    model_name = args.model_name
45
46
    # yml path generate.
47
    # If model_name is not set, script will check all of the models.
48
    if model_name is not None:
49
        yml_list = [(model_name, f'configs/{model_name}/{model_name}.yml')]
50
    else:
51
        # check all
52
        yml_list = [(x, f'configs/{x}/{x}.yml') for x in os.listdir('configs/')
53
                    if x != '_base_']
54
55
    logger = get_root_logger(log_file='url_check.log', log_level=logging.ERROR)
56
57
    for model_name, yml_path in yml_list:
58
        # Default yaml loader unsafe.
59
        model_infos = yml.load(
60
            open(yml_path, 'r'), Loader=yml.CLoader)['Models']
61
        for model_info in model_infos:
62
            config_name = model_info['Name']
63
            checkpoint_url = model_info['Weights']
64
            # checkpoint url check
65
            status_code, flag = check_url(checkpoint_url)
66
            if flag:
67
                logger.info(f'checkpoint | {config_name} | {checkpoint_url} | '
68
                            f'{status_code} valid')
69
            else:
70
                logger.error(
71
                    f'checkpoint | {config_name} | {checkpoint_url} | '
72
                    f'{status_code} | error')
73
            # log_json check
74
            checkpoint_name = checkpoint_url.split('/')[-1]
75
            model_time = '-'.join(checkpoint_name.split('-')[:-1]).replace(
76
                f'{config_name}_', '')
77
            # two style of log_json name
78
            # use '_' to link model_time (will be deprecated)
79
            log_json_url_1 = f'https://download.openmmlab.com/mmsegmentation/v0.5/{model_name}/{config_name}/{config_name}_{model_time}.log.json'  # noqa
80
            status_code_1, flag_1 = check_url(log_json_url_1)
81
            # use '-' to link model_time
82
            log_json_url_2 = f'https://download.openmmlab.com/mmsegmentation/v0.5/{model_name}/{config_name}/{config_name}-{model_time}.log.json'  # noqa
83
            status_code_2, flag_2 = check_url(log_json_url_2)
84
            if flag_1 or flag_2:
85
                if flag_1:
86
                    logger.info(
87
                        f'log.json | {config_name} | {log_json_url_1} | '
88
                        f'{status_code_1} | valid')
89
                else:
90
                    logger.info(
91
                        f'log.json | {config_name} | {log_json_url_2} | '
92
                        f'{status_code_2} | valid')
93
            else:
94
                logger.error(
95
                    f'log.json | {config_name} | {log_json_url_1} & '
96
                    f'{log_json_url_2} | {status_code_1} & {status_code_2} | '
97
                    'error')
98
99
100
if __name__ == '__main__':
101
    main()