import os
import sys
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np
from .bbox import *
from .utils import *
class Evaluator:
def GetPascalVOCMetrics(
self,
boundingboxes,
IOUThreshold=0.5,
method=MethodAveragePrecision.EveryPointInterpolation):
"""Get the metrics used by the VOC Pascal 2012 challenge.
Get
Args:
boundingboxes: Object of the class BoundingBoxes representing ground truth and detected
bounding boxes;
IOUThreshold: IOU threshold indicating which detections will be considered TP or FP
(default value = 0.5);
method (default = EveryPointInterpolation): It can be calculated as the implementation
in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
or EveryPointInterpolation" (ElevenPointInterpolation);
Returns:
A list of dictionaries. Each dictionary contains information and metrics of each class.
The keys of each dictionary are:
dict['class']: class representing the current dictionary;
dict['precision']: array with the precision values;
dict['recall']: array with the recall values;
dict['AP']: average precision;
dict['interpolated precision']: interpolated precision values;
dict['interpolated recall']: interpolated recall values;
dict['total positives']: total number of ground truth positives;
dict['total TP']: total number of True Positive detections;
dict['total FP']: total number of False Positive detections;
"""
ret = [
] # list containing metrics (precision, recall, average precision) of each class
# List with all ground truths (Ex: [imageName,class,confidence=1, (bb
# coordinates XYX2Y2)])
groundTruths = []
# List with all detections (Ex: [imageName,class,confidence,(bb
# coordinates XYX2Y2)])
detections = []
# Get all classes
classes = []
# Loop through all bounding boxes and separate them into GTs and
# detections
for bb in boundingboxes.getBoundingBoxes():
# [imageName, class, confidence, (bb coordinates XYX2Y2)]
if bb.getBBType() == BBType.GroundTruth:
groundTruths.append([
bb.getImageName(),
bb.getClassId(), 1,
bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
])
else:
detections.append([
bb.getImageName(),
bb.getClassId(),
bb.getConfidence(),
bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
])
# get class
if bb.getClassId() not in classes:
classes.append(bb.getClassId())
classes = sorted(classes)
# Precision x Recall is obtained individually by each class
# Loop through by classes
for c in classes:
# Get only detection of class c
dects = []
[dects.append(d) for d in detections if d[1] == c]
# Get only ground truths of class c, use filename as key
gts = {}
npos = 0
for g in groundTruths:
if g[1] == c:
npos += 1
gts[g[0]] = gts.get(g[0], []) + [g]
# sort detections by decreasing confidence
dects = sorted(dects, key=lambda conf: conf[2], reverse=True)
TP = np.zeros(len(dects))
FP = np.zeros(len(dects))
score = np.zeros(len(dects))
# create dictionary with amount of gts for each image
det = {key: np.zeros(len(gts[key])) for key in gts}
# Loop through detections
for d in range(len(dects)):
score[d] = dects[d][2]
# Find ground truth image
gt = gts[dects[d][0]] if dects[d][0] in gts else []
iouMax = sys.float_info.min
for j in range(len(gt)):
iou = Evaluator.iou(dects[d][3], gt[j][3])
if iou > iouMax:
iouMax = iou
jmax = j
# Assign detection as true positive/don't care/false positive
if iouMax >= IOUThreshold:
if det[dects[d][0]][jmax] == 0:
TP[d] = 1 # count as true positive
det[dects[d][0]][jmax] = 1 # flag as already 'seen'
else:
FP[d] = 1 # count as false positive
# - A detected "cat" is overlaped with a GT "cat" with IOU >= IOUThreshold.
else:
FP[d] = 1 # count as false positive
# compute precision, recall and average precision
acc_FP = np.cumsum(FP)
acc_TP = np.cumsum(TP)
rec = acc_TP / npos
prec = np.divide(acc_TP, (acc_FP + acc_TP))
# Depending on the method, call the right implementation
if method == MethodAveragePrecision.EveryPointInterpolation:
[ap, mpre, mrec, ii] = Evaluator.CalculateAveragePrecision(
rec, prec)
else:
[ap, mpre, mrec, _] = Evaluator.ElevenPointInterpolatedAP(
rec, prec)
# add class result in the dictionary to be returned
r = {
'class': c,
'precision': prec,
'recall': rec,
'AP': ap,
'interpolated precision': mpre,
'interpolated recall': mrec,
'total positives': npos,
'total TP': np.sum(TP),
'total FP': np.sum(FP),
'score': score,
}
ret.append(r)
return ret
def PlotPrecisionRecallCurve(
self,
boundingBoxes,
IOUThreshold=0.5,
method=MethodAveragePrecision.EveryPointInterpolation,
showAP=False,
showInterpolatedPrecision=False,
savePath=None,
showGraphic=True):
"""PlotPrecisionRecallCurve
Plot the Precision x Recall curve for a given class.
Args:
boundingBoxes: Object of the class BoundingBoxes representing ground truth and detected
bounding boxes;
IOUThreshold (optional): IOU threshold indicating which detections will be considered
TP or FP (default value = 0.5);
method (default = EveryPointInterpolation): It can be calculated as the implementation
in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
or EveryPointInterpolation" (ElevenPointInterpolation).
showAP (optional): if True, the average precision value will be shown in the title of
the graph (default = False);
showInterpolatedPrecision (optional): if True, it will show in the plot the interpolated
precision (default = False);
savePath (optional): if informed, the plot will be saved as an image in this path
(ex: /home/mywork/ap.png) (default = None);
showGraphic (optional): if True, the plot will be shown (default = True)
Returns:
A list of dictionaries. Each dictionary contains information and metrics of each class.
The keys of each dictionary are:
dict['class']: class representing the current dictionary;
dict['precision']: array with the precision values;
dict['recall']: array with the recall values;
dict['AP']: average precision;
dict['interpolated precision']: interpolated precision values;
dict['interpolated recall']: interpolated recall values;
dict['total positives']: total number of ground truth positives;
dict['total TP']: total number of True Positive detections;
dict['total FP']: total number of False Negative detections;
"""
results = self.GetPascalVOCMetrics(boundingBoxes, IOUThreshold, method)
result = None
# Each resut represents a class
for result in results:
if result is None:
raise IOError('Error: Class %d could not be found.' % classId)
classId = result['class']
precision = result['precision']
recall = result['recall']
average_precision = result['AP']
mpre = result['interpolated precision']
mrec = result['interpolated recall']
npos = result['total positives']
total_tp = result['total TP']
total_fp = result['total FP']
plt.close()
if showInterpolatedPrecision:
if method == MethodAveragePrecision.EveryPointInterpolation:
plt.plot(
mrec,
mpre,
'--r',
label='Interpolated precision (every point)')
elif method == MethodAveragePrecision.ElevenPointInterpolation:
nrec = []
nprec = []
for idx in range(len(mrec)):
r = mrec[idx]
if r not in nrec:
idxEq = np.argwhere(mrec == r)
nrec.append(r)
nprec.append(max([mpre[int(id)] for id in idxEq]))
plt.plot(
nrec,
nprec,
'or',
label='11-point interpolated precision')
plt.plot(recall, precision, label='Precision')
plt.xlabel('recall')
plt.ylabel('precision')
if showAP:
ap_str = "{0:.2f}%".format(average_precision * 100)
# ap_str = "{0:.4f}%".format(average_precision * 100)
plt.title(
'Precision x Recall curve \nClass: %s, AP: %s' %
(str(classId), ap_str))
else:
plt.title('Precision x Recall curve \nClass: %s' % str(classId))
plt.legend(shadow=True)
plt.grid()
if savePath is not None:
plt.savefig(os.path.join(savePath, f'{classId}.png'))
if showGraphic is True:
plt.show()
# plt.waitforbuttonpress()
plt.pause(0.05)
return results
@staticmethod
def CalculateAveragePrecision(rec, prec):
mrec = []
mrec.append(0)
[mrec.append(e) for e in rec]
mrec.append(1)
mpre = []
mpre.append(0)
[mpre.append(e) for e in prec]
mpre.append(0)
for i in range(len(mpre) - 1, 0, -1):
mpre[i - 1] = max(mpre[i - 1], mpre[i])
ii = []
for i in range(len(mrec) - 1):
if mrec[1 + i] != mrec[i]:
ii.append(i + 1)
ap = 0
for i in ii:
ap = ap + np.sum((mrec[i] - mrec[i - 1]) * mpre[i])
return [ap, mpre[0:len(mpre) - 1], mrec[0:len(mpre) - 1], ii]
@staticmethod
# 11-point interpolated average precision
def ElevenPointInterpolatedAP(rec, prec):
# def CalculateAveragePrecision2(rec, prec):
mrec = []
[mrec.append(e) for e in rec]
mpre = []
[mpre.append(e) for e in prec]
recallValues = np.linspace(0, 1, 11)
recallValues = list(recallValues[::-1])
rhoInterp = []
recallValid = []
# For each recallValues (0, 0.1, 0.2, ... , 1)
for r in recallValues:
# Obtain all recall values higher or equal than r
argGreaterRecalls = np.argwhere(mrec[:] >= r)
pmax = 0
# If there are recalls above r
if argGreaterRecalls.size != 0:
pmax = max(mpre[argGreaterRecalls.min():])
recallValid.append(r)
rhoInterp.append(pmax)
# By definition AP = sum(max(precision whose recall is above r))/11
ap = sum(rhoInterp) / 11
# Generating values for the plot
rvals = []
rvals.append(recallValid[0])
[rvals.append(e) for e in recallValid]
rvals.append(0)
pvals = []
pvals.append(0)
[pvals.append(e) for e in rhoInterp]
pvals.append(0)
cc = []
for i in range(len(rvals)):
p = (rvals[i], pvals[i - 1])
if p not in cc:
cc.append(p)
p = (rvals[i], pvals[i])
if p not in cc:
cc.append(p)
recallValues = [i[0] for i in cc]
rhoInterp = [i[1] for i in cc]
return [ap, rhoInterp, recallValues, None]
# For each detections, calculate IOU with reference
@staticmethod
def _getAllIOUs(reference, detections):
ret = []
bbReference = reference.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
for d in detections:
bb = d.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
iou = Evaluator.iou(bbReference, bb)
ret.append((iou, reference, d)) # iou, reference, detection
return sorted(ret, key=lambda i: i[0], reverse=True)
@staticmethod
def iou(boxA, boxB):
if Evaluator._boxesIntersect(boxA, boxB) is False:
return 0
interArea = Evaluator._getIntersectionArea(boxA, boxB)
union = Evaluator._getUnionAreas(boxA, boxB, interArea=interArea)
iou = interArea / union
assert iou >= 0
return iou
@staticmethod
def _boxesIntersect(boxA, boxB):
if boxA[0] > boxB[2]:
return False # boxA is right of boxB
if boxB[0] > boxA[2]:
return False # boxA is left of boxB
if boxA[3] < boxB[1]:
return False # boxA is above boxB
if boxA[1] > boxB[3]:
return False # boxA is below boxB
return True
@staticmethod
def _getIntersectionArea(boxA, boxB):
xA = max(boxA[0], boxB[0])
yA = max(boxA[1], boxB[1])
xB = min(boxA[2], boxB[2])
yB = min(boxA[3], boxB[3])
# intersection area
return (xB - xA + 1) * (yB - yA + 1)
@staticmethod
def _getUnionAreas(boxA, boxB, interArea=None):
area_A = Evaluator._getArea(boxA)
area_B = Evaluator._getArea(boxB)
if interArea is None:
interArea = Evaluator._getIntersectionArea(boxA, boxB)
return float(area_A + area_B - interArea)
@staticmethod
def _getArea(box):
return (box[2] - box[0] + 1) * (box[3] - box[1] + 1)
Datasets
Models
