|
a |
|
b/utils/pycocoevalcap/eval.py |
|
|
1 |
__author__ = 'tylin' |
|
|
2 |
from .tokenizer.ptbtokenizer import PTBTokenizer |
|
|
3 |
from .bleu.bleu import Bleu |
|
|
4 |
from .meteor.meteor import Meteor |
|
|
5 |
from .rouge.rouge import Rouge |
|
|
6 |
from .cider.cider import Cider |
|
|
7 |
|
|
|
8 |
class COCOEvalCap: |
|
|
9 |
def __init__(self, coco, cocoRes): |
|
|
10 |
self.evalImgs = [] |
|
|
11 |
self.eval = {} |
|
|
12 |
self.imgToEval = {} |
|
|
13 |
self.coco = coco |
|
|
14 |
self.cocoRes = cocoRes |
|
|
15 |
self.params = {'image_id': cocoRes.getImgIds()} |
|
|
16 |
|
|
|
17 |
def evaluate(self): |
|
|
18 |
imgIds = self.params['image_id'] |
|
|
19 |
# imgIds = self.coco.getImgIds() |
|
|
20 |
gts = {} |
|
|
21 |
res = {} |
|
|
22 |
for imgId in imgIds: |
|
|
23 |
gts[imgId] = self.coco.imgToAnns[imgId] |
|
|
24 |
res[imgId] = self.cocoRes.imgToAnns[imgId] |
|
|
25 |
|
|
|
26 |
# ================================================= |
|
|
27 |
# Set up scorers |
|
|
28 |
# ================================================= |
|
|
29 |
print('tokenization...') |
|
|
30 |
tokenizer = PTBTokenizer() |
|
|
31 |
gts = tokenizer.tokenize(gts) |
|
|
32 |
res = tokenizer.tokenize(res) |
|
|
33 |
|
|
|
34 |
# ================================================= |
|
|
35 |
# Set up scorers |
|
|
36 |
# ================================================= |
|
|
37 |
print('setting up scorers...') |
|
|
38 |
scorers = [ |
|
|
39 |
(Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]), |
|
|
40 |
(Meteor(),"METEOR"), |
|
|
41 |
(Rouge(), "ROUGE_L"), |
|
|
42 |
(Cider(), "CIDEr") |
|
|
43 |
] |
|
|
44 |
|
|
|
45 |
# ================================================= |
|
|
46 |
# Compute scores |
|
|
47 |
# ================================================= |
|
|
48 |
eval = {} |
|
|
49 |
for scorer, method in scorers: |
|
|
50 |
print('computing %s score...'%(scorer.method())) |
|
|
51 |
score, scores = scorer.compute_score(gts, res) |
|
|
52 |
if type(method) == list: |
|
|
53 |
for sc, scs, m in zip(score, scores, method): |
|
|
54 |
self.setEval(sc, m) |
|
|
55 |
self.setImgToEvalImgs(scs, imgIds, m) |
|
|
56 |
print("%s: %0.3f"%(m, sc)) |
|
|
57 |
else: |
|
|
58 |
self.setEval(score, method) |
|
|
59 |
self.setImgToEvalImgs(scores, imgIds, method) |
|
|
60 |
print("%s: %0.3f"%(method, score)) |
|
|
61 |
self.setEvalImgs() |
|
|
62 |
|
|
|
63 |
def setEval(self, score, method): |
|
|
64 |
self.eval[method] = score |
|
|
65 |
|
|
|
66 |
def setImgToEvalImgs(self, scores, imgIds, method): |
|
|
67 |
for imgId, score in zip(imgIds, scores): |
|
|
68 |
if not imgId in self.imgToEval: |
|
|
69 |
self.imgToEval[imgId] = {} |
|
|
70 |
self.imgToEval[imgId]["image_id"] = imgId |
|
|
71 |
self.imgToEval[imgId][method] = score |
|
|
72 |
|
|
|
73 |
def setEvalImgs(self): |
|
|
74 |
self.evalImgs = [eval for imgId, eval in self.imgToEval.items()] |