[7829e6]: / reproducibility / utils / cacher.py

Download this file

75 lines (56 with data), 2.1 kB

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import os
import hashlib
import numpy as np
def get_cache_name(name: str, path: str):
"""
Generates the cache name of the file using sha256.
:param name:
:param path:
:return:
"""
key = name+path
cache_folder = os.environ["PC_CACHE_FOLDER"]
m = hashlib.sha256()
m.update(key.encode('utf-8'))
save_path = os.path.join(cache_folder, m.hexdigest())
return save_path
def cache_hit_or_miss(name: str, path: str):
save_path = get_cache_name(name, path)
if os.path.exists(save_path):
return np.load(save_path)
else:
return None
def cache_numpy_object(npa, name, path):
key = name+path
cache_folder = os.environ["PC_CACHE_FOLDER"]
m = hashlib.sha256()
m.update(key.encode('utf-8'))
save_path = os.path.join(cache_folder, m.hexdigest())
with open(f"{save_path}", 'wb') as f:
np.save(f, npa)
###############################################################
# below are new codes
###############################################################
def get_savepath(name, path):
modelname, dataset_name = name.split('img')
dataset_name = dataset_name.split('.csv')[0]
cache_folder = os.environ["PC_CACHE_FOLDER"]
cache_subfolder_data = os.path.join(cache_folder, dataset_name, modelname)
os.makedirs(cache_subfolder_data, exist_ok=True)
if modelname == 'plip':
path = os.path.basename(path)
save_path = os.path.join(cache_subfolder_data, path)
return save_path
def cache_hit_or_miss_raw_filename(name: str, path: str):
save_path = get_savepath(name, path)
if os.path.exists(save_path):
print('[CACHE] Found existed embedding.')
return np.load(save_path)
else:
print('[CACHE] No existed embedding found. Need to generate embedding first.')
return None
def cache_numpy_object_raw_filename(npa, name, path):
save_path = get_savepath(name, path)
print(f"[CACHE] Saving embedding. Name: {name}, Path: {path}, Save path: {save_path}")
with open(f"{save_path}", 'wb') as f:
np.save(f, npa)