|
a |
|
b/yolov5/utils/downloads.py |
|
|
1 |
# YOLOv5 🚀 by Ultralytics, GPL-3.0 license |
|
|
2 |
""" |
|
|
3 |
Download utils |
|
|
4 |
""" |
|
|
5 |
|
|
|
6 |
import os |
|
|
7 |
import platform |
|
|
8 |
import subprocess |
|
|
9 |
import time |
|
|
10 |
import urllib |
|
|
11 |
from pathlib import Path |
|
|
12 |
from zipfile import ZipFile |
|
|
13 |
|
|
|
14 |
import requests |
|
|
15 |
import torch |
|
|
16 |
|
|
|
17 |
|
|
|
18 |
def gsutil_getsize(url=''): |
|
|
19 |
# gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du |
|
|
20 |
s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8') |
|
|
21 |
return eval(s.split(' ')[0]) if len(s) else 0 # bytes |
|
|
22 |
|
|
|
23 |
|
|
|
24 |
def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''): |
|
|
25 |
# Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes |
|
|
26 |
file = Path(file) |
|
|
27 |
assert_msg = f"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}" |
|
|
28 |
try: # url1 |
|
|
29 |
print(f'Downloading {url} to {file}...') |
|
|
30 |
torch.hub.download_url_to_file(url, str(file)) |
|
|
31 |
assert file.exists() and file.stat().st_size > min_bytes, assert_msg # check |
|
|
32 |
except Exception as e: # url2 |
|
|
33 |
file.unlink(missing_ok=True) # remove partial downloads |
|
|
34 |
print(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...') |
|
|
35 |
os.system(f"curl -L '{url2 or url}' -o '{file}' --retry 3 -C -") # curl download, retry and resume on fail |
|
|
36 |
finally: |
|
|
37 |
if not file.exists() or file.stat().st_size < min_bytes: # check |
|
|
38 |
file.unlink(missing_ok=True) # remove partial downloads |
|
|
39 |
print(f"ERROR: {assert_msg}\n{error_msg}") |
|
|
40 |
print('') |
|
|
41 |
|
|
|
42 |
|
|
|
43 |
def attempt_download(file, repo='ultralytics/yolov5'): # from utils.downloads import *; attempt_download() |
|
|
44 |
# Attempt file download if does not exist |
|
|
45 |
file = Path(str(file).strip().replace("'", '')) |
|
|
46 |
|
|
|
47 |
if not file.exists(): |
|
|
48 |
# URL specified |
|
|
49 |
name = Path(urllib.parse.unquote(str(file))).name # decode '%2F' to '/' etc. |
|
|
50 |
if str(file).startswith(('http:/', 'https:/')): # download |
|
|
51 |
url = str(file).replace(':/', '://') # Pathlib turns :// -> :/ |
|
|
52 |
name = name.split('?')[0] # parse authentication https://url.com/file.txt?auth... |
|
|
53 |
safe_download(file=name, url=url, min_bytes=1E5) |
|
|
54 |
return name |
|
|
55 |
|
|
|
56 |
# GitHub assets |
|
|
57 |
file.parent.mkdir(parents=True, exist_ok=True) # make parent dir (if required) |
|
|
58 |
try: |
|
|
59 |
response = requests.get(f'https://api.github.com/repos/{repo}/releases/latest').json() # github api |
|
|
60 |
assets = [x['name'] for x in response['assets']] # release assets, i.e. ['yolov5s.pt', 'yolov5m.pt', ...] |
|
|
61 |
tag = response['tag_name'] # i.e. 'v1.0' |
|
|
62 |
except: # fallback plan |
|
|
63 |
assets = ['yolov5n.pt', 'yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt', |
|
|
64 |
'yolov5n6.pt', 'yolov5s6.pt', 'yolov5m6.pt', 'yolov5l6.pt', 'yolov5x6.pt'] |
|
|
65 |
try: |
|
|
66 |
tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1] |
|
|
67 |
except: |
|
|
68 |
tag = 'v6.0' # current release |
|
|
69 |
|
|
|
70 |
if name in assets: |
|
|
71 |
safe_download(file, |
|
|
72 |
url=f'https://github.com/{repo}/releases/download/{tag}/{name}', |
|
|
73 |
# url2=f'https://storage.googleapis.com/{repo}/ckpt/{name}', # backup url (optional) |
|
|
74 |
min_bytes=1E5, |
|
|
75 |
error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/') |
|
|
76 |
|
|
|
77 |
return str(file) |
|
|
78 |
|
|
|
79 |
|
|
|
80 |
def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m', file='tmp.zip'): |
|
|
81 |
# Downloads a file from Google Drive. from yolov5.utils.downloads import *; gdrive_download() |
|
|
82 |
t = time.time() |
|
|
83 |
file = Path(file) |
|
|
84 |
cookie = Path('cookie') # gdrive cookie |
|
|
85 |
print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='') |
|
|
86 |
file.unlink(missing_ok=True) # remove existing file |
|
|
87 |
cookie.unlink(missing_ok=True) # remove existing cookie |
|
|
88 |
|
|
|
89 |
# Attempt file download |
|
|
90 |
out = "NUL" if platform.system() == "Windows" else "/dev/null" |
|
|
91 |
os.system(f'curl -c ./cookie -s -L "drive.google.com/uc?export=download&id={id}" > {out}') |
|
|
92 |
if os.path.exists('cookie'): # large file |
|
|
93 |
s = f'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm={get_token()}&id={id}" -o {file}' |
|
|
94 |
else: # small file |
|
|
95 |
s = f'curl -s -L -o {file} "drive.google.com/uc?export=download&id={id}"' |
|
|
96 |
r = os.system(s) # execute, capture return |
|
|
97 |
cookie.unlink(missing_ok=True) # remove existing cookie |
|
|
98 |
|
|
|
99 |
# Error check |
|
|
100 |
if r != 0: |
|
|
101 |
file.unlink(missing_ok=True) # remove partial |
|
|
102 |
print('Download error ') # raise Exception('Download error') |
|
|
103 |
return r |
|
|
104 |
|
|
|
105 |
# Unzip if archive |
|
|
106 |
if file.suffix == '.zip': |
|
|
107 |
print('unzipping... ', end='') |
|
|
108 |
ZipFile(file).extractall(path=file.parent) # unzip |
|
|
109 |
file.unlink() # remove zip |
|
|
110 |
|
|
|
111 |
print(f'Done ({time.time() - t:.1f}s)') |
|
|
112 |
return r |
|
|
113 |
|
|
|
114 |
|
|
|
115 |
def get_token(cookie="./cookie"): |
|
|
116 |
with open(cookie) as f: |
|
|
117 |
for line in f: |
|
|
118 |
if "download" in line: |
|
|
119 |
return line.split()[-1] |
|
|
120 |
return "" |
|
|
121 |
|
|
|
122 |
# Google utils: https://cloud.google.com/storage/docs/reference/libraries ---------------------------------------------- |
|
|
123 |
# |
|
|
124 |
# |
|
|
125 |
# def upload_blob(bucket_name, source_file_name, destination_blob_name): |
|
|
126 |
# # Uploads a file to a bucket |
|
|
127 |
# # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python |
|
|
128 |
# |
|
|
129 |
# storage_client = storage.Client() |
|
|
130 |
# bucket = storage_client.get_bucket(bucket_name) |
|
|
131 |
# blob = bucket.blob(destination_blob_name) |
|
|
132 |
# |
|
|
133 |
# blob.upload_from_filename(source_file_name) |
|
|
134 |
# |
|
|
135 |
# print('File {} uploaded to {}.'.format( |
|
|
136 |
# source_file_name, |
|
|
137 |
# destination_blob_name)) |
|
|
138 |
# |
|
|
139 |
# |
|
|
140 |
# def download_blob(bucket_name, source_blob_name, destination_file_name): |
|
|
141 |
# # Uploads a blob from a bucket |
|
|
142 |
# storage_client = storage.Client() |
|
|
143 |
# bucket = storage_client.get_bucket(bucket_name) |
|
|
144 |
# blob = bucket.blob(source_blob_name) |
|
|
145 |
# |
|
|
146 |
# blob.download_to_filename(destination_file_name) |
|
|
147 |
# |
|
|
148 |
# print('Blob {} downloaded to {}.'.format( |
|
|
149 |
# source_blob_name, |
|
|
150 |
# destination_file_name)) |