From 769d2ada0eff5d6e5662051219621aca7fd96342 Mon Sep 17 00:00:00 2001 From: lluis Date: Thu, 6 Aug 2015 11:53:46 +0200 Subject: [PATCH 1/2] Changes SVT recognition evaluation to be Case Insensitive (according to the standard evaluation protocol). This makes the benchmark obtained mean-f1 score increase from 0.23 to 0.27 --- modules/datasets/samples/tr_svt_benchmark.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/datasets/samples/tr_svt_benchmark.cpp b/modules/datasets/samples/tr_svt_benchmark.cpp index be2d74295..2f903165a 100644 --- a/modules/datasets/samples/tr_svt_benchmark.cpp +++ b/modules/datasets/samples/tr_svt_benchmark.cpp @@ -244,6 +244,8 @@ int main(int argc, char *argv[]) continue; } + std::transform(words[j].begin(), words[j].end(), words[j].begin(), ::toupper); + if (find(example->lex.begin(), example->lex.end(), words[j]) == example->lex.end()) { continue; From be927d962a5b8ec9fa0f75dafeeef982e7862807 Mon Sep 17 00:00:00 2001 From: lluis Date: Thu, 6 Aug 2015 11:59:53 +0200 Subject: [PATCH 2/2] Report the total f1 score (0.37) and not the mean-f1. This is in accordance with the standard SVT evaluation protocol and allows for comparison with other published results. --- modules/datasets/samples/tr_svt_benchmark.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/datasets/samples/tr_svt_benchmark.cpp b/modules/datasets/samples/tr_svt_benchmark.cpp index 2f903165a..4c7b2d1b0 100644 --- a/modules/datasets/samples/tr_svt_benchmark.cpp +++ b/modules/datasets/samples/tr_svt_benchmark.cpp @@ -288,18 +288,18 @@ int main(int argc, char *argv[]) f1Each.push_back(f1); } - /*double p = 1.0*returnedCorrectNum/returnedNum; + double p = 1.0*returnedCorrectNum/returnedNum; double r = 1.0*returnedCorrectNum/correctNum; double f1 = 2*(p*r)/(p+r); - printf("f1: %f\n", f1);*/ + printf("f1: %f\n", f1); - double f1 = 0.0; + /*double f1 = 0.0; for (vector::iterator it=f1Each.begin(); it!=f1Each.end(); ++it) { f1 += *it; } f1 /= f1Each.size(); - printf("mean f1: %f\n", f1); + printf("mean f1: %f\n", f1);*/ return 0; }