Diff of /utils/downloads.py [000000] .. [190ca4]

Switch to unified view

a b/utils/downloads.py
1
# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license
2
"""
3
Download utils
4
"""
5
6
import logging
7
import subprocess
8
import urllib
9
from pathlib import Path
10
11
import requests
12
import torch
13
14
15
def is_url(url, check=True):
16
    # Check if string is URL and check if URL exists
17
    try:
18
        url = str(url)
19
        result = urllib.parse.urlparse(url)
20
        assert all([result.scheme, result.netloc])  # check if is url
21
        return (urllib.request.urlopen(url).getcode() == 200) if check else True  # check if exists online
22
    except (AssertionError, urllib.request.HTTPError):
23
        return False
24
25
26
def gsutil_getsize(url=''):
27
    # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
28
    output = subprocess.check_output(['gsutil', 'du', url], shell=True, encoding='utf-8')
29
    if output:
30
        return int(output.split()[0])
31
    return 0
32
33
34
def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
35
    # Return downloadable file size in bytes
36
    response = requests.head(url, allow_redirects=True)
37
    return int(response.headers.get('content-length', -1))
38
39
40
def curl_download(url, filename, *, silent: bool = False) -> bool:
41
    """
42
    Download a file from a url to a filename using curl.
43
    """
44
    silent_option = 'sS' if silent else ''  # silent
45
    proc = subprocess.run([
46
        'curl',
47
        '-#',
48
        f'-{silent_option}L',
49
        url,
50
        '--output',
51
        filename,
52
        '--retry',
53
        '9',
54
        '-C',
55
        '-', ])
56
    return proc.returncode == 0
57
58
59
def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
60
    # Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes
61
    from utils.general import LOGGER
62
63
    file = Path(file)
64
    assert_msg = f"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}"
65
    try:  # url1
66
        LOGGER.info(f'Downloading {url} to {file}...')
67
        torch.hub.download_url_to_file(url, str(file), progress=LOGGER.level <= logging.INFO)
68
        assert file.exists() and file.stat().st_size > min_bytes, assert_msg  # check
69
    except Exception as e:  # url2
70
        if file.exists():
71
            file.unlink()  # remove partial downloads
72
        LOGGER.info(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...')
73
        # curl download, retry and resume on fail
74
        curl_download(url2 or url, file)
75
    finally:
76
        if not file.exists() or file.stat().st_size < min_bytes:  # check
77
            if file.exists():
78
                file.unlink()  # remove partial downloads
79
            LOGGER.info(f'ERROR: {assert_msg}\n{error_msg}')
80
        LOGGER.info('')
81
82
83
def attempt_download(file, repo='ultralytics/yolov5', release='v7.0'):
84
    # Attempt file download from GitHub release assets if not found locally. release = 'latest', 'v7.0', etc.
85
    from utils.general import LOGGER
86
87
    def github_assets(repository, version='latest'):
88
        # Return GitHub repo tag (i.e. 'v7.0') and assets (i.e. ['yolov5s.pt', 'yolov5m.pt', ...])
89
        if version != 'latest':
90
            version = f'tags/{version}'  # i.e. tags/v7.0
91
        response = requests.get(f'https://api.github.com/repos/{repository}/releases/{version}').json()  # github api
92
        return response['tag_name'], [x['name'] for x in response['assets']]  # tag, assets
93
94
    file = Path(str(file).strip().replace("'", ''))
95
    if not file.exists():
96
        # URL specified
97
        name = Path(urllib.parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
98
        if str(file).startswith(('http:/', 'https:/')):  # download
99
            url = str(file).replace(':/', '://')  # Pathlib turns :// -> :/
100
            file = name.split('?')[0]  # parse authentication https://url.com/file.txt?auth...
101
            if Path(file).is_file():
102
                LOGGER.info(f'Found {url} locally at {file}')  # file already exists
103
            else:
104
                safe_download(file=file, url=url, min_bytes=1E5)
105
            return file
106
107
        # GitHub assets
108
        assets = [f'yolov5{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')]  # default
109
        try:
110
            tag, assets = github_assets(repo, release)
111
        except Exception:
112
            try:
113
                tag, assets = github_assets(repo)  # latest release
114
            except Exception:
115
                try:
116
                    tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1]
117
                except Exception:
118
                    tag = release
119
120
        if name in assets:
121
            file.parent.mkdir(parents=True, exist_ok=True)  # make parent dir (if required)
122
            safe_download(file,
123
                          url=f'https://github.com/{repo}/releases/download/{tag}/{name}',
124
                          min_bytes=1E5,
125
                          error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/{tag}')
126
127
    return str(file)