diff --git a/modules/core/perf/perf_core_arithm.cpp b/modules/core/perf/perf_core_arithm.cpp index 849423d483..0927134261 100644 --- a/modules/core/perf/perf_core_arithm.cpp +++ b/modules/core/perf/perf_core_arithm.cpp @@ -52,14 +52,12 @@ PERF_TEST_P__CORE_ARITHM(add, TYPICAL_MATS_CORE_ARITHM) PERF_TEST_P__CORE_ARITHM(subtract, TYPICAL_MATS_CORE_ARITHM) PERF_TEST_P__CORE_ARITHM(min, TYPICAL_MATS_CORE_ARITHM) PERF_TEST_P__CORE_ARITHM(max, TYPICAL_MATS_CORE_ARITHM) -PERF_TEST_P__CORE_ARITHM(absdiff, TYPICAL_MATS_CORE_ARITHM) PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_and, TYPICAL_MATS_BITW_ARITHM) PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_or, TYPICAL_MATS_BITW_ARITHM) PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_xor, TYPICAL_MATS_BITW_ARITHM) PERF_TEST_P__CORE_ARITHM_SCALAR(add, TYPICAL_MATS_CORE_ARITHM) PERF_TEST_P__CORE_ARITHM_SCALAR(subtract, TYPICAL_MATS_CORE_ARITHM) -PERF_TEST_P__CORE_ARITHM_SCALAR(absdiff, TYPICAL_MATS_CORE_ARITHM) #ifdef ANDROID PERF_TEST(convert, cvRound) @@ -75,3 +73,41 @@ PERF_TEST(convert, cvRound) } } #endif + +PERF_TEST_P(Size_MatType, core_arithm__absdiff, TYPICAL_MATS_CORE_ARITHM) +{ + Size sz = std::tr1::get<0>(GetParam()); + int type = std::tr1::get<1>(GetParam()); + cv::Mat a = Mat(sz, type); + cv::Mat b = Mat(sz, type); + cv::Mat c = Mat(sz, type); + + declare.in(a, b, WARMUP_RNG) + .out(c); + + TEST_CYCLE(100) absdiff(a,b, c); + +#if CV_SSE2 //see ticket 1529: absdiff can be without saturation if SSE is enabled + if (CV_MAT_DEPTH(type) != CV_32S) +#endif + SANITY_CHECK(c, 1e-8); +} + +PERF_TEST_P(Size_MatType, core_arithm__absdiff__Scalar, TYPICAL_MATS_CORE_ARITHM) +{ + Size sz = std::tr1::get<0>(GetParam()); + int type = std::tr1::get<1>(GetParam()); + cv::Mat a = Mat(sz, type); + cv::Scalar b; + cv::Mat c = Mat(sz, type); + + declare.in(a, b, WARMUP_RNG) + .out(c); + + TEST_CYCLE(100) absdiff(a,b, c); + +#if CV_SSE2 //see ticket 1529: absdiff can be without saturation if SSE is enabled + if (CV_MAT_DEPTH(type) != CV_32S) +#endif + SANITY_CHECK(c, 1e-8); +} diff --git a/modules/core/perf/perf_stat.cpp b/modules/core/perf/perf_stat.cpp index f5f7ff9843..7a688eb42b 100644 --- a/modules/core/perf/perf_stat.cpp +++ b/modules/core/perf/perf_stat.cpp @@ -21,7 +21,7 @@ PERF_TEST_P( Size_MatType, sum, TYPICAL_MATS ) TEST_CYCLE(100) { s = sum(arr); } - SANITY_CHECK(s, 1e-6); + SANITY_CHECK(s, 1e-6, ERROR_RELATIVE); } @@ -89,7 +89,7 @@ PERF_TEST_P( Size_MatType_NormType, norm, TEST_CYCLE(100) { n = norm(src1, normType); } - SANITY_CHECK(n, 1e-5); + SANITY_CHECK(n, 1e-6, ERROR_RELATIVE); } @@ -116,7 +116,7 @@ PERF_TEST_P( Size_MatType_NormType, norm_mask, TEST_CYCLE(100) { n = norm(src1, normType, mask); } - SANITY_CHECK(n, 1e-5); + SANITY_CHECK(n, 1e-6, ERROR_RELATIVE); } @@ -143,7 +143,7 @@ PERF_TEST_P( Size_MatType_NormType, norm2, TEST_CYCLE(100) { n = norm(src1, src2, normType); } - SANITY_CHECK(n, 1e-5); + SANITY_CHECK(n, 1e-5, ERROR_RELATIVE); } @@ -171,7 +171,7 @@ PERF_TEST_P( Size_MatType_NormType, norm2_mask, TEST_CYCLE(100) { n = norm(src1, src2, normType, mask); } - SANITY_CHECK(n, 1e-5); + SANITY_CHECK(n, 1e-5, ERROR_RELATIVE); } @@ -258,8 +258,8 @@ PERF_TEST_P( Size_MatType_NormType, normalize_32f, declare.in(src, WARMUP_RNG).out(dst); TEST_CYCLE(100) { normalize(src, dst, alpha, 0., normType, CV_32F); } - - SANITY_CHECK(dst, 1e-6); + + SANITY_CHECK(dst, 1e-6, ERROR_RELATIVE); } @@ -404,7 +404,7 @@ PERF_TEST_P( Size_MatType_ROp, reduceR, TEST_CYCLE(100) { reduce(src, vec, 0, reduceOp, ddepth); } - SANITY_CHECK(vec); + SANITY_CHECK(vec, 1); } /* @@ -431,6 +431,6 @@ PERF_TEST_P( Size_MatType_ROp, reduceC, declare.in(src, WARMUP_RNG); TEST_CYCLE(100) { reduce(src, vec, 1, reduceOp, ddepth); } - - SANITY_CHECK(vec); + + SANITY_CHECK(vec, 1); } diff --git a/modules/ts/include/opencv2/ts/ts_perf.hpp b/modules/ts/include/opencv2/ts/ts_perf.hpp index 2e8f2bef2b..106c9d153d 100644 --- a/modules/ts/include/opencv2/ts/ts_perf.hpp +++ b/modules/ts/include/opencv2/ts/ts_perf.hpp @@ -149,13 +149,19 @@ CV_ENUM(MatDepth, CV_8U, CV_8S, CV_16U, CV_16S, CV_32S, CV_32F, CV_64F, CV_USRTY /*****************************************************************************************\ * Regression control utility for performance testing * \*****************************************************************************************/ +enum ERROR_TYPE +{ + ERROR_ABSOLUTE = 0, + ERROR_RELATIVE = 1 +}; + class CV_EXPORTS Regression { public: - static Regression& add(const std::string& name, cv::InputArray array, double eps = DBL_EPSILON); + static Regression& add(const std::string& name, cv::InputArray array, double eps = DBL_EPSILON, ERROR_TYPE err = ERROR_ABSOLUTE); static void Init(const std::string& testSuitName, const std::string& ext = ".xml"); - Regression& operator() (const std::string& name, cv::InputArray array, double eps = DBL_EPSILON); + Regression& operator() (const std::string& name, cv::InputArray array, double eps = DBL_EPSILON, ERROR_TYPE err = ERROR_ABSOLUTE); private: static Regression& instance(); @@ -181,8 +187,8 @@ private: void init(const std::string& testSuitName, const std::string& ext); void write(cv::InputArray array); void write(cv::Mat m); - void verify(cv::FileNode node, cv::InputArray array, double eps); - void verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname); + void verify(cv::FileNode node, cv::InputArray array, double eps, ERROR_TYPE err); + void verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname, ERROR_TYPE err); }; #define SANITY_CHECK(array, ...) ::perf::Regression::add(#array, array , ## __VA_ARGS__) diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index d1d41e80bc..a29ff46987 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -43,9 +43,9 @@ Regression& Regression::instance() return single; } -Regression& Regression::add(const std::string& name, cv::InputArray array, double eps) +Regression& Regression::add(const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err) { - return instance()(name, array, eps); + return instance()(name, array, eps, err); } void Regression::Init(const std::string& testSuitName, const std::string& ext) @@ -202,13 +202,25 @@ void Regression::write(cv::Mat m) write() << "val" << getElem(m, y, x, cn) << "}"; } -void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname) +static double evalEps(double expected, double actual, double _eps, ERROR_TYPE err) +{ + if (err == ERROR_ABSOLUTE) + return _eps; + else if (err == ERROR_RELATIVE) + return std::max(std::abs(expected), std::abs(actual)) * err; + return 0; +} + +void Regression::verify(cv::FileNode node, cv::Mat actual, double _eps, std::string argname, ERROR_TYPE err) { double actual_min, actual_max; cv::minMaxLoc(actual, &actual_min, &actual_max); + double eps = evalEps((double)node["min"], actual_min, _eps, err); ASSERT_NEAR((double)node["min"], actual_min, eps) << " " << argname << " has unexpected minimal value"; + + eps = evalEps((double)node["max"], actual_max, _eps, err); ASSERT_NEAR((double)node["max"], actual_max, eps) << " " << argname << " has unexpected maximal value"; @@ -218,6 +230,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri << " " << argname << " has unexpected number of columns"; ASSERT_EQ((int)last["y"], actual.rows - 1) << " " << argname << " has unexpected number of rows"; + + eps = evalEps((double)last["val"], actualLast, _eps, err); ASSERT_NEAR((double)last["val"], actualLast, eps) << " " << argname << " has unexpected value of last element"; @@ -225,6 +239,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri int x1 = rng1["x"]; int y1 = rng1["y"]; int cn1 = rng1["cn"]; + + eps = evalEps((double)rng1["val"], getElem(actual, y1, x1, cn1), _eps, err); ASSERT_NEAR((double)rng1["val"], getElem(actual, y1, x1, cn1), eps) << " " << argname << " has unexpected value of ["<< x1 << ":" << y1 << ":" << cn1 <<"] element"; @@ -232,6 +248,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri int x2 = rng2["x"]; int y2 = rng2["y"]; int cn2 = rng2["cn"]; + + eps = evalEps((double)rng2["val"], getElem(actual, y2, x2, cn2), _eps, err); ASSERT_NEAR((double)rng2["val"], getElem(actual, y2, x2, cn2), eps) << " " << argname << " has unexpected value of ["<< x2 << ":" << y2 << ":" << cn2 <<"] element"; } @@ -263,7 +281,31 @@ void Regression::write(cv::InputArray array) } } -void Regression::verify(cv::FileNode node, cv::InputArray array, double eps) +static int countViolations(const cv::Mat& expected, const cv::Mat& actual, const cv::Mat& diff, double eps, double* max_violation = 0, double* max_allowed = 0) +{ + cv::Mat diff64f; + diff.reshape(1).convertTo(diff64f, CV_64F); + + cv::Mat expected_abs = cv::abs(expected.reshape(1)); + cv::Mat actual_abs = cv::abs(actual.reshape(1)); + cv::Mat maximum, mask; + cv::max(expected_abs, actual_abs, maximum); + cv::multiply(maximum, cv::Vec(eps), maximum, CV_64F); + cv::compare(diff64f, maximum, mask, cv::CMP_GT); + + int v = cv::countNonZero(mask); + + if (v > 0 && max_violation != 0 && max_allowed != 0) + { + int loc[10]; + cv::minMaxIdx(maximum, 0, max_allowed, 0, loc, mask); + *max_violation = diff64f.at(loc[1], loc[0]); + } + + return v; +} + +void Regression::verify(cv::FileNode node, cv::InputArray array, double eps, ERROR_TYPE err) { ASSERT_EQ((int)node["kind"], array.kind()) << " Argument \"" << node.name() << "\" has unexpected kind"; ASSERT_EQ((int)node["type"], array.type()) << " Argument \"" << node.name() << "\" has unexpected type"; @@ -280,7 +322,7 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps) { ASSERT_LE((size_t)26, actual.total() * (size_t)actual.channels()) << " \"" << node.name() << "[" << idx << "]\" has unexpected number of elements"; - verify(node, actual, eps, cv::format("%s[%d]", node.name().c_str(), idx)); + verify(node, actual, eps, cv::format("%s[%d]", node.name().c_str(), idx), err); } else { @@ -292,12 +334,26 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps) cv::Mat diff; cv::absdiff(expected, actual, diff); - if (!cv::checkRange(diff, true, 0, 0, eps)) + + if (err == ERROR_ABSOLUTE) { - double max; - cv::minMaxLoc(diff, 0, &max); - FAIL() << " Difference (=" << max << ") between argument \"" - << node.name() << "[" << idx << "]\" and expected value is bugger than " << eps; + if (!cv::checkRange(diff, true, 0, 0, eps)) + { + double max; + cv::minMaxLoc(diff.reshape(1), 0, &max); + FAIL() << " Absolute difference (=" << max << ") between argument \"" + << node.name() << "[" << idx << "]\" and expected value is bugger than " << eps; + } + } + else if (err == ERROR_RELATIVE) + { + double maxv, maxa; + int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa); + if (violations > 0) + { + FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \"" + << node.name() << "[" << idx << "]\" and expected value is bugger than " << eps << " in " << violations << " points"; + } } } } @@ -307,7 +363,7 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps) { ASSERT_LE((size_t)26, array.total() * (size_t)array.channels()) << " Argument \"" << node.name() << "\" has unexpected number of elements"; - verify(node, array.getMat(), eps, "Argument " + node.name()); + verify(node, array.getMat(), eps, "Argument " + node.name(), err); } else { @@ -320,18 +376,32 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps) cv::Mat diff; cv::absdiff(expected, actual, diff); - if (!cv::checkRange(diff, true, 0, 0, eps)) + + if (err == ERROR_ABSOLUTE) + { + if (!cv::checkRange(diff, true, 0, 0, eps)) + { + double max; + cv::minMaxLoc(diff.reshape(1), 0, &max); + FAIL() << " Difference (=" << max << ") between argument \"" << node.name() + << "\" and expected value is bugger than " << eps; + } + } + else if (err == ERROR_RELATIVE) { - double max; - cv::minMaxLoc(diff, 0, &max); - FAIL() << " Difference (=" << max << ") between argument \"" << node.name() - << "\" and expected value is bugger than " << eps; + double maxv, maxa; + int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa); + if (violations > 0) + { + FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \"" << node.name() + << "\" and expected value is bugger than " << eps << " in " << violations << " points"; + } } } } } -Regression& Regression::operator() (const std::string& name, cv::InputArray array, double eps) +Regression& Regression::operator() (const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err) { std::string nodename = getCurrentTestNodeName(); @@ -356,7 +426,7 @@ Regression& Regression::operator() (const std::string& name, cv::InputArray arra if (!this_arg.isMap()) ADD_FAILURE() << " No regression data for " << name << " argument"; else - verify(this_arg, array, eps); + verify(this_arg, array, eps, err); } return *this; }