|
a |
|
b/metrics/metrics.py |
|
|
1 |
''' |
|
|
2 |
Copyright (c) Microsoft Corporation. All rights reserved. |
|
|
3 |
Licensed under the MIT License. |
|
|
4 |
''' |
|
|
5 |
|
|
|
6 |
import SimpleITK as sitk |
|
|
7 |
import numpy as np |
|
|
8 |
import cc3d |
|
|
9 |
|
|
|
10 |
#%% |
|
|
11 |
def get_3darray_from_niftipath( |
|
|
12 |
path: str, |
|
|
13 |
) -> np.ndarray: |
|
|
14 |
"""Get a numpy array of a Nifti image using the filepath |
|
|
15 |
|
|
|
16 |
Args: |
|
|
17 |
path (str): path of the Nifti file |
|
|
18 |
|
|
|
19 |
Returns: |
|
|
20 |
np.ndarray: 3D numpy array for the image |
|
|
21 |
""" |
|
|
22 |
image = sitk.ReadImage(path) |
|
|
23 |
array = np.transpose(sitk.GetArrayFromImage(image), (2,1,0)) |
|
|
24 |
return array |
|
|
25 |
|
|
|
26 |
def calculate_patient_level_lesion_suvmean_suvmax( |
|
|
27 |
ptarray: np.ndarray, |
|
|
28 |
maskarray: np.ndarray, |
|
|
29 |
marker: str = 'SUVmean' |
|
|
30 |
) -> np.float64: |
|
|
31 |
"""Function to return the lesion SUVmean or SUVmax for all lesions in |
|
|
32 |
a 3D PET image using the corresponding 3D segmentation mask |
|
|
33 |
|
|
|
34 |
Args: |
|
|
35 |
ptarray (np.ndarray): numpy ndarray for 3D PET image |
|
|
36 |
maskarray (np.ndarray): numpy ndarray for 3D mask image |
|
|
37 |
marker (str, optional): Whether you want to calculate SUVmean or SUVmax . |
|
|
38 |
Defaults to 'SUVmean'. |
|
|
39 |
|
|
|
40 |
Returns: |
|
|
41 |
np.float64: patient-level SUVmean or SUVmax |
|
|
42 |
""" |
|
|
43 |
prod = np.multiply(ptarray, maskarray) |
|
|
44 |
num_nonzero_voxels = len(np.nonzero(maskarray)[0]) |
|
|
45 |
|
|
|
46 |
if num_nonzero_voxels == 0: |
|
|
47 |
return 0.0 |
|
|
48 |
else: |
|
|
49 |
if marker == 'SUVmean': |
|
|
50 |
return np.sum(prod)/num_nonzero_voxels |
|
|
51 |
elif marker == 'SUVmax': |
|
|
52 |
return np.max(prod) |
|
|
53 |
|
|
|
54 |
#%% |
|
|
55 |
def calculate_patient_level_tmtv( |
|
|
56 |
maskarray: np.ndarray, |
|
|
57 |
spacing: tuple |
|
|
58 |
) -> np.float64: |
|
|
59 |
"""Function to return the total metabolic tumor volume (TMTV) in cm^3 using |
|
|
60 |
3D mask containing 0s for background and 1s for lesions/tumors |
|
|
61 |
Args: |
|
|
62 |
maskarray (np.ndarray): numpy ndarray for 3D mask image |
|
|
63 |
|
|
|
64 |
Returns: |
|
|
65 |
np.float64: |
|
|
66 |
""" |
|
|
67 |
voxel_volume_cc = np.prod(spacing)/1000 # voxel volume in cm^3 |
|
|
68 |
|
|
|
69 |
num_lesion_voxels = len(np.nonzero(maskarray)[0]) |
|
|
70 |
tmtv_cc = voxel_volume_cc*num_lesion_voxels |
|
|
71 |
return tmtv_cc |
|
|
72 |
|
|
|
73 |
#%% |
|
|
74 |
|
|
|
75 |
def calculate_patient_level_lesion_count( |
|
|
76 |
maskarray: np.ndarray, |
|
|
77 |
) -> int: |
|
|
78 |
"""Function to return the total number of lesions using the 3D segmentation mask |
|
|
79 |
Args: |
|
|
80 |
maskarray (np.ndarray): numpy ndarray for 3D mask image |
|
|
81 |
|
|
|
82 |
Returns: |
|
|
83 |
int: _description_ |
|
|
84 |
""" |
|
|
85 |
_, num_lesions = cc3d.connected_components(maskarray, connectivity=18, return_N=True) |
|
|
86 |
return num_lesions |
|
|
87 |
|
|
|
88 |
#%% |
|
|
89 |
def calculate_patient_level_tlg( |
|
|
90 |
ptarray: np.ndarray, |
|
|
91 |
maskarray: np.ndarray, |
|
|
92 |
spacing: tuple |
|
|
93 |
) -> np.float64: |
|
|
94 |
"""Function to return the total lesion glycolysis (TLG) using a 3D PET image |
|
|
95 |
and the corresponding 3D segmentation mask (containing 0s for background and |
|
|
96 |
1s for lesion/tumor) |
|
|
97 |
TLG = SUV1*V1 + SUV2*V2 + ... + SUVn*Vn, where SUV1...SUVn are the SUVmean |
|
|
98 |
values of lesions 1...n with volumes V1...Vn, respectively |
|
|
99 |
|
|
|
100 |
Args: |
|
|
101 |
ptarray (np.ndarray): numpy ndarray for 3D PET image |
|
|
102 |
maskarray (np.ndarray): numpy ndarray for 3D mask image |
|
|
103 |
|
|
|
104 |
Returns: |
|
|
105 |
np.float64: total lesion glycolysis in cm^3 (assuming SUV is unitless) |
|
|
106 |
""" |
|
|
107 |
voxel_volume_cc = np.prod(spacing)/1000 # voxel volume in cm^3 |
|
|
108 |
|
|
|
109 |
labels_out, num_lesions = cc3d.connected_components(maskarray, connectivity=18, return_N=True) |
|
|
110 |
if num_lesions == 0: |
|
|
111 |
return 0.0 |
|
|
112 |
else: |
|
|
113 |
_, lesion_num_voxels = np.unique(labels_out, return_counts=True) |
|
|
114 |
lesion_num_voxels = lesion_num_voxels[1:] |
|
|
115 |
lesion_mtvs = voxel_volume_cc*lesion_num_voxels |
|
|
116 |
lesion_suvmeans = [] |
|
|
117 |
|
|
|
118 |
for i in range(1, num_lesions+1): |
|
|
119 |
mask = np.zeros_like(labels_out) |
|
|
120 |
mask[labels_out == i] = 1 |
|
|
121 |
prod = np.multiply(mask, ptarray) |
|
|
122 |
num_nonzero_voxels = len(np.nonzero(mask)[0]) |
|
|
123 |
lesion_suvmeans.append(np.sum(prod)/num_nonzero_voxels) |
|
|
124 |
|
|
|
125 |
tlg = np.sum(np.multiply(lesion_mtvs, lesion_suvmeans)) |
|
|
126 |
return tlg |
|
|
127 |
#%% |
|
|
128 |
def calculate_patient_level_dissemination( |
|
|
129 |
maskarray: np.ndarray, |
|
|
130 |
spacing: tuple |
|
|
131 |
) -> np.float64: |
|
|
132 |
"""Function to return the tumor dissemination (Dmax) using 3D segmentation mask |
|
|
133 |
Dmax = max possible distance between any two foreground voxels in a patient; |
|
|
134 |
these two voxels can come form the same lesions (in case of one lesion) |
|
|
135 |
or from different lesions (in case of multiple lesions) |
|
|
136 |
|
|
|
137 |
Args: |
|
|
138 |
maskarray (np.ndarray): numpy array for 3D mask image |
|
|
139 |
|
|
|
140 |
Returns: |
|
|
141 |
np.float64: dissemination value in cm |
|
|
142 |
""" |
|
|
143 |
maskarray = maskarray.astype(np.int8) |
|
|
144 |
nonzero_voxels = np.argwhere(maskarray == 1) |
|
|
145 |
distances = np.sqrt(np.sum(((nonzero_voxels[:, None] - nonzero_voxels) * spacing)**2, axis=2)) |
|
|
146 |
farthest_indices = np.unravel_index(np.argmax(distances), distances.shape) |
|
|
147 |
dmax = distances[farthest_indices]/10 # converting to cm |
|
|
148 |
del maskarray |
|
|
149 |
del nonzero_voxels |
|
|
150 |
del distances |
|
|
151 |
return dmax |
|
|
152 |
|
|
|
153 |
#%% |
|
|
154 |
def calculate_patient_level_dice_score( |
|
|
155 |
gtarray: np.ndarray, |
|
|
156 |
predarray: np.ndarray, |
|
|
157 |
) -> np.float64: |
|
|
158 |
"""Function to return the Dice similarity coefficient (Dice score) between |
|
|
159 |
2 segmentation masks (containing 0s for background and 1s for lesions/tumors) |
|
|
160 |
|
|
|
161 |
Args: |
|
|
162 |
maskarray_1 (np.ndarray): numpy ndarray for the first mask |
|
|
163 |
maskarray_2 (np.ndarray): numpy ndarray for the second mask |
|
|
164 |
|
|
|
165 |
Returns: |
|
|
166 |
np.float64: Dice score |
|
|
167 |
""" |
|
|
168 |
dice_score = 2.0*np.sum(predarray[gtarray == 1])/(np.sum(gtarray) + np.sum(predarray)) |
|
|
169 |
return dice_score |
|
|
170 |
#%% |
|
|
171 |
def calculate_patient_level_iou( |
|
|
172 |
gtarray: np.ndarray, |
|
|
173 |
predarray: np.ndarray, |
|
|
174 |
) -> np.float64: |
|
|
175 |
"""Function to return the Intersection-over-Union (IoU) between |
|
|
176 |
2 segmentation masks (containing 0s for background and 1s for lesions/tumors) |
|
|
177 |
|
|
|
178 |
Args: |
|
|
179 |
maskarray_1 (np.ndarray): numpy ndarray for the first mask |
|
|
180 |
maskarray_2 (np.ndarray): numpy ndarray for the second mask |
|
|
181 |
|
|
|
182 |
Returns: |
|
|
183 |
np.float64: Dice score |
|
|
184 |
""" |
|
|
185 |
intersection = np.sum(predarray[gtarray == 1]) |
|
|
186 |
union = np.sum(gtarray) + np.sum(predarray) - intersection |
|
|
187 |
iou = intersection/union |
|
|
188 |
return iou |
|
|
189 |
|
|
|
190 |
def calculate_patient_level_intersection( |
|
|
191 |
gtarray: np.ndarray, |
|
|
192 |
predarray: np.ndarray, |
|
|
193 |
) -> np.float64: |
|
|
194 |
"""Function to return the Intersection etween |
|
|
195 |
2 segmentation masks (containing 0s for background and 1s for lesions/tumors) |
|
|
196 |
|
|
|
197 |
Args: |
|
|
198 |
maskarray_1 (np.ndarray): numpy ndarray for the first mask |
|
|
199 |
maskarray_2 (np.ndarray): numpy ndarray for the second mask |
|
|
200 |
|
|
|
201 |
Returns: |
|
|
202 |
np.float64: Dice score |
|
|
203 |
""" |
|
|
204 |
intersection = np.sum(predarray[gtarray == 1]) |
|
|
205 |
return intersection |
|
|
206 |
#%% |
|
|
207 |
|
|
|
208 |
def calculate_patient_level_false_positive_volume( |
|
|
209 |
gtarray: np.ndarray, |
|
|
210 |
predarray: np.ndarray, |
|
|
211 |
spacing: tuple |
|
|
212 |
) -> np.float64: |
|
|
213 |
# compute number of voxels of false positive connected components in prediction mask |
|
|
214 |
pred_connected_components = cc3d.connected_components(predarray, connectivity=18) |
|
|
215 |
|
|
|
216 |
false_positive = 0 |
|
|
217 |
for idx in range(1,pred_connected_components.max()+1): |
|
|
218 |
comp_mask = np.isin(pred_connected_components, idx) |
|
|
219 |
if (comp_mask*gtarray).sum() == 0: |
|
|
220 |
false_positive += comp_mask.sum() |
|
|
221 |
|
|
|
222 |
voxel_volume_cc = np.prod(spacing)/1000 |
|
|
223 |
return false_positive*voxel_volume_cc |
|
|
224 |
|
|
|
225 |
#%% |
|
|
226 |
def calculate_patient_level_false_negative_volume( |
|
|
227 |
gtarray: np.ndarray, |
|
|
228 |
predarray: np.ndarray, |
|
|
229 |
spacing: tuple |
|
|
230 |
) -> np.float64: |
|
|
231 |
# compute number of voxels of false negative connected components (of the ground truth mask) in the prediction mask |
|
|
232 |
gt_connected_components = cc3d.connected_components(gtarray, connectivity=18) |
|
|
233 |
|
|
|
234 |
false_negative = 0 |
|
|
235 |
for idx in range(1,gt_connected_components.max()+1): |
|
|
236 |
comp_mask = np.isin(gt_connected_components, idx) |
|
|
237 |
if (comp_mask*predarray).sum() == 0: |
|
|
238 |
false_negative += comp_mask.sum() |
|
|
239 |
|
|
|
240 |
voxel_volume_cc = np.prod(spacing)/1000 |
|
|
241 |
return false_negative*voxel_volume_cc |
|
|
242 |
|
|
|
243 |
# %% |
|
|
244 |
def is_suvmax_detected( |
|
|
245 |
gtarray: np.ndarray, |
|
|
246 |
predarray: np.ndarray, |
|
|
247 |
ptarray: np.ndarray, |
|
|
248 |
) -> bool: |
|
|
249 |
prod = np.multiply(gtarray, ptarray) |
|
|
250 |
max_index = np.unravel_index(np.argmax(prod), prod.shape) |
|
|
251 |
if predarray[max_index] == 1: |
|
|
252 |
return True |
|
|
253 |
else: |
|
|
254 |
return False |
|
|
255 |
|
|
|
256 |
|
|
|
257 |
def calculate_patient_level_tp_fp_fn( |
|
|
258 |
gtarray: np.ndarray, |
|
|
259 |
predarray: np.ndarray, |
|
|
260 |
criterion: str, |
|
|
261 |
threshold: np.float64 = None, |
|
|
262 |
ptarray: np.ndarray = None, |
|
|
263 |
) -> (int, int, int): |
|
|
264 |
"""Calculate patient-level TP, FP, and FN (for detection based metrics) |
|
|
265 |
via 3 criteria: |
|
|
266 |
|
|
|
267 |
criterion1: A predicted lesion is TP if any one of it's foreground voxels |
|
|
268 |
overlaps with GT foreground. A predicted lesions that doesn't overlap with any |
|
|
269 |
GT foreground is FP. As soon as a lesion is predicted as TP, it is removed |
|
|
270 |
from the set of GT lesions. The lesions that remain in the end in the GT lesions |
|
|
271 |
are FN. `criterion1` is the weakest detection criterion. |
|
|
272 |
|
|
|
273 |
criterion2: A predicted lesion is TP if more than `threshold`% of it's volume |
|
|
274 |
overlaps with foreground GT. A predicted lesion is FP if it overlap fraction |
|
|
275 |
with foreground GT is between 0% and `threshold`%. As soon as a lesion is |
|
|
276 |
predicted as TP, it is removed from the set of GT lesions. The lesions that |
|
|
277 |
remain in the end in the GT lesions are FN. `criterion2` can be hard or weak |
|
|
278 |
criterion based on the value of `threshold`. |
|
|
279 |
|
|
|
280 |
criterion3: A predicted lesion is TP if it overlaps with one the the GT lesion's |
|
|
281 |
SUVmax voxel, hence this criterion requires the use of PET data (`ptarray`). A |
|
|
282 |
predicted lesion that doesn't overlap with any GT lesion's SUVmax voxel is |
|
|
283 |
considered FP. As soon as a lesion is predicted as TP, it is removed from the |
|
|
284 |
set of GT lesions. The lesions that remain in the end in the GT lesions are FN. |
|
|
285 |
`criterion3` is likely an easy criterion since a network is more likely to segment |
|
|
286 |
high(er)-uptake regions`. |
|
|
287 |
|
|
|
288 |
Args: |
|
|
289 |
int (_type_): _description_ |
|
|
290 |
int (_type_): _description_ |
|
|
291 |
gtarray (_type_, optional): _description_. Defaults to None, ptarray: np.ndarray = None, )->(int. |
|
|
292 |
""" |
|
|
293 |
|
|
|
294 |
gtarray_labeled_mask, num_lesions_gt = cc3d.connected_components(gtarray, connectivity=18, return_N=True) |
|
|
295 |
predarray_labeled_mask, num_lesions_pred = cc3d.connected_components(predarray, connectivity=18, return_N=True) |
|
|
296 |
gt_lesions_list = list(np.arange(1, num_lesions_gt+1)) |
|
|
297 |
#initial values for TP, FP, FN |
|
|
298 |
TP = 0 |
|
|
299 |
FP = 0 |
|
|
300 |
FN = num_lesions_gt |
|
|
301 |
|
|
|
302 |
if criterion == 'criterion1': |
|
|
303 |
FN = 0 # for this criterion we are counting the number of FPs from 0 onwards, hence the reassignment |
|
|
304 |
for i in range(1, num_lesions_pred+1): |
|
|
305 |
pred_lesion_mask = np.where(predarray_labeled_mask == i, 1, 0) |
|
|
306 |
if np.any(pred_lesion_mask & (gtarray_labeled_mask > 0)): |
|
|
307 |
TP += 1 |
|
|
308 |
else: |
|
|
309 |
FP += 1 |
|
|
310 |
for j in range(1, num_lesions_gt+1): |
|
|
311 |
gt_lesion_mask = np.where(gtarray_labeled_mask == j, 1, 0) |
|
|
312 |
if not np.any(gt_lesion_mask & (predarray_labeled_mask > 0)): |
|
|
313 |
FN += 1 |
|
|
314 |
|
|
|
315 |
elif criterion == 'criterion2': |
|
|
316 |
for i in range(1, num_lesions_pred+1): |
|
|
317 |
max_iou = 0 |
|
|
318 |
match_gt_lesion = None |
|
|
319 |
pred_lesion_mask = np.where(predarray_labeled_mask == i, 1, 0) |
|
|
320 |
for j in range(1, num_lesions_gt+1): |
|
|
321 |
gt_lesion_mask = np.where(gtarray_labeled_mask == j, 1, 0) |
|
|
322 |
iou = calculate_patient_level_iou(gt_lesion_mask, pred_lesion_mask) |
|
|
323 |
if iou > max_iou: |
|
|
324 |
max_iou = iou |
|
|
325 |
match_gt_lesion = j |
|
|
326 |
if max_iou >= threshold: |
|
|
327 |
TP += 1 |
|
|
328 |
gt_lesions_list.remove(match_gt_lesion) |
|
|
329 |
else: |
|
|
330 |
FP += 1 |
|
|
331 |
FN = len(gt_lesions_list) |
|
|
332 |
|
|
|
333 |
elif criterion == 'criterion3': |
|
|
334 |
for i in range(1, num_lesions_pred+1): |
|
|
335 |
max_iou = 0 |
|
|
336 |
match_gt_lesion = None |
|
|
337 |
pred_lesion_mask = np.where(predarray_labeled_mask == i, 1, 0) |
|
|
338 |
for j in range(1, num_lesions_gt+1): |
|
|
339 |
gt_lesion_mask = np.where(gtarray_labeled_mask == j, 1, 0) |
|
|
340 |
iou = calculate_patient_level_iou(gt_lesion_mask, pred_lesion_mask) |
|
|
341 |
if iou > max_iou: |
|
|
342 |
max_iou = iou |
|
|
343 |
match_gt_lesion = j |
|
|
344 |
|
|
|
345 |
# match_gt_lesion has been defined with has the maximum iou with pred lesion i |
|
|
346 |
arr_gt_lesion = np.where(gtarray_labeled_mask == match_gt_lesion, 1, 0) |
|
|
347 |
if is_suvmax_detected(arr_gt_lesion, pred_lesion_mask, ptarray): |
|
|
348 |
TP += 1 |
|
|
349 |
gt_lesions_list.remove(match_gt_lesion) |
|
|
350 |
else: |
|
|
351 |
FP += 1 |
|
|
352 |
|
|
|
353 |
FN = len(gt_lesions_list) |
|
|
354 |
|
|
|
355 |
else: |
|
|
356 |
print('Invalid criterion. Choose between criterion1, criterion2, or criterion3') |
|
|
357 |
return |
|
|
358 |
|
|
|
359 |
return TP, FP, FN |
|
|
360 |
|