Diff of /yolov5/utils/downloads.py [000000] .. [f26a44]

Switch to unified view

a b/yolov5/utils/downloads.py
1
# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
2
"""
3
Download utils
4
"""
5
6
import os
7
import platform
8
import subprocess
9
import time
10
import urllib
11
from pathlib import Path
12
from zipfile import ZipFile
13
14
import requests
15
import torch
16
17
18
def gsutil_getsize(url=''):
19
    # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
20
    s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')
21
    return eval(s.split(' ')[0]) if len(s) else 0  # bytes
22
23
24
def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
25
    # Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes
26
    file = Path(file)
27
    assert_msg = f"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}"
28
    try:  # url1
29
        print(f'Downloading {url} to {file}...')
30
        torch.hub.download_url_to_file(url, str(file))
31
        assert file.exists() and file.stat().st_size > min_bytes, assert_msg  # check
32
    except Exception as e:  # url2
33
        file.unlink(missing_ok=True)  # remove partial downloads
34
        print(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...')
35
        os.system(f"curl -L '{url2 or url}' -o '{file}' --retry 3 -C -")  # curl download, retry and resume on fail
36
    finally:
37
        if not file.exists() or file.stat().st_size < min_bytes:  # check
38
            file.unlink(missing_ok=True)  # remove partial downloads
39
            print(f"ERROR: {assert_msg}\n{error_msg}")
40
        print('')
41
42
43
def attempt_download(file, repo='ultralytics/yolov5'):  # from utils.downloads import *; attempt_download()
44
    # Attempt file download if does not exist
45
    file = Path(str(file).strip().replace("'", ''))
46
47
    if not file.exists():
48
        # URL specified
49
        name = Path(urllib.parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
50
        if str(file).startswith(('http:/', 'https:/')):  # download
51
            url = str(file).replace(':/', '://')  # Pathlib turns :// -> :/
52
            name = name.split('?')[0]  # parse authentication https://url.com/file.txt?auth...
53
            safe_download(file=name, url=url, min_bytes=1E5)
54
            return name
55
56
        # GitHub assets
57
        file.parent.mkdir(parents=True, exist_ok=True)  # make parent dir (if required)
58
        try:
59
            response = requests.get(f'https://api.github.com/repos/{repo}/releases/latest').json()  # github api
60
            assets = [x['name'] for x in response['assets']]  # release assets, i.e. ['yolov5s.pt', 'yolov5m.pt', ...]
61
            tag = response['tag_name']  # i.e. 'v1.0'
62
        except:  # fallback plan
63
            assets = ['yolov5n.pt', 'yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt',
64
                      'yolov5n6.pt', 'yolov5s6.pt', 'yolov5m6.pt', 'yolov5l6.pt', 'yolov5x6.pt']
65
            try:
66
                tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1]
67
            except:
68
                tag = 'v6.0'  # current release
69
70
        if name in assets:
71
            safe_download(file,
72
                          url=f'https://github.com/{repo}/releases/download/{tag}/{name}',
73
                          # url2=f'https://storage.googleapis.com/{repo}/ckpt/{name}',  # backup url (optional)
74
                          min_bytes=1E5,
75
                          error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/')
76
77
    return str(file)
78
79
80
def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m', file='tmp.zip'):
81
    # Downloads a file from Google Drive. from yolov5.utils.downloads import *; gdrive_download()
82
    t = time.time()
83
    file = Path(file)
84
    cookie = Path('cookie')  # gdrive cookie
85
    print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='')
86
    file.unlink(missing_ok=True)  # remove existing file
87
    cookie.unlink(missing_ok=True)  # remove existing cookie
88
89
    # Attempt file download
90
    out = "NUL" if platform.system() == "Windows" else "/dev/null"
91
    os.system(f'curl -c ./cookie -s -L "drive.google.com/uc?export=download&id={id}" > {out}')
92
    if os.path.exists('cookie'):  # large file
93
        s = f'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm={get_token()}&id={id}" -o {file}'
94
    else:  # small file
95
        s = f'curl -s -L -o {file} "drive.google.com/uc?export=download&id={id}"'
96
    r = os.system(s)  # execute, capture return
97
    cookie.unlink(missing_ok=True)  # remove existing cookie
98
99
    # Error check
100
    if r != 0:
101
        file.unlink(missing_ok=True)  # remove partial
102
        print('Download error ')  # raise Exception('Download error')
103
        return r
104
105
    # Unzip if archive
106
    if file.suffix == '.zip':
107
        print('unzipping... ', end='')
108
        ZipFile(file).extractall(path=file.parent)  # unzip
109
        file.unlink()  # remove zip
110
111
    print(f'Done ({time.time() - t:.1f}s)')
112
    return r
113
114
115
def get_token(cookie="./cookie"):
116
    with open(cookie) as f:
117
        for line in f:
118
            if "download" in line:
119
                return line.split()[-1]
120
    return ""
121
122
# Google utils: https://cloud.google.com/storage/docs/reference/libraries ----------------------------------------------
123
#
124
#
125
# def upload_blob(bucket_name, source_file_name, destination_blob_name):
126
#     # Uploads a file to a bucket
127
#     # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python
128
#
129
#     storage_client = storage.Client()
130
#     bucket = storage_client.get_bucket(bucket_name)
131
#     blob = bucket.blob(destination_blob_name)
132
#
133
#     blob.upload_from_filename(source_file_name)
134
#
135
#     print('File {} uploaded to {}.'.format(
136
#         source_file_name,
137
#         destination_blob_name))
138
#
139
#
140
# def download_blob(bucket_name, source_blob_name, destination_file_name):
141
#     # Uploads a blob from a bucket
142
#     storage_client = storage.Client()
143
#     bucket = storage_client.get_bucket(bucket_name)
144
#     blob = bucket.blob(source_blob_name)
145
#
146
#     blob.download_to_filename(destination_file_name)
147
#
148
#     print('Blob {} downloaded to {}.'.format(
149
#         source_blob_name,
150
#         destination_file_name))