From 86fb9f8409bd79ac35febb67f3b2863398d15b32 Mon Sep 17 00:00:00 2001 From: Owen Healy Date: Wed, 18 Mar 2015 21:11:33 -0400 Subject: [PATCH] Modify findTransformECC to support a mask of pixels to consider Tests of the mask are also included. This is useful for registering a non-square image against a non-square template. This also needs to relax a sanity check as per https://github.com/Itseez/opencv/pull/3851 --- .../video/include/opencv2/video/tracking.hpp | 4 +- modules/video/perf/perf_ecc.cpp | 2 +- modules/video/src/ecc.cpp | 29 +++++-- modules/video/test/test_ecc.cpp | 83 +++++++++++++++++++ 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 90be72ea2d..c725a7caab 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -278,6 +278,7 @@ order to provide an image similar to templateImage, same type as temlateImage. criteria.epsilon defines the threshold of the increment in the correlation coefficient between two iterations (a negative criteria.epsilon makes criteria.maxcount the only termination criterion). Default values are shown in the declaration above. +@param inputMask An optional mask to indicate valid values of inputImage. The function estimates the optimum transformation (warpMatrix) with respect to ECC criterion (@cite EP08), that is @@ -309,7 +310,8 @@ estimateRigidTransform, findHomography */ CV_EXPORTS_W double findTransformECC( InputArray templateImage, InputArray inputImage, InputOutputArray warpMatrix, int motionType = MOTION_AFFINE, - TermCriteria criteria = TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 50, 0.001)); + TermCriteria criteria = TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 50, 0.001), + InputArray inputMask = noArray()); /** @brief Kalman filter class. diff --git a/modules/video/perf/perf_ecc.cpp b/modules/video/perf/perf_ecc.cpp index 72410cf576..c706caa07f 100644 --- a/modules/video/perf/perf_ecc.cpp +++ b/modules/video/perf/perf_ecc.cpp @@ -67,5 +67,5 @@ PERF_TEST_P(TransformationType, findTransformECC, /*testing::ValuesIn(MotionType findTransformECC(templateImage, img, warpMat, transform_type, TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 5, -1)); } - SANITY_CHECK(warpMat, 1e-3); + SANITY_CHECK(warpMat, 3e-3); } diff --git a/modules/video/src/ecc.cpp b/modules/video/src/ecc.cpp index 8c5fbee790..d527f8f890 100644 --- a/modules/video/src/ecc.cpp +++ b/modules/video/src/ecc.cpp @@ -317,7 +317,8 @@ double cv::findTransformECC(InputArray templateImage, InputArray inputImage, InputOutputArray warpMatrix, int motionType, - TermCriteria criteria) + TermCriteria criteria, + InputArray inputMask) { @@ -397,13 +398,26 @@ double cv::findTransformECC(InputArray templateImage, Mat templateFloat = Mat(hs, ws, CV_32F);// to store the (smoothed) template Mat imageFloat = Mat(hd, wd, CV_32F);// to store the (smoothed) input image Mat imageWarped = Mat(hs, ws, CV_32F);// to store the warped zero-mean input image - Mat allOnes = Mat::ones(hd, wd, CV_8U); //to use it for mask warping Mat imageMask = Mat(hs, ws, CV_8U); //to store the final mask + Mat inputMaskMat = inputMask.getMat(); + //to use it for mask warping + Mat preMask = + inputMaskMat.empty() ? Mat::ones(hd, wd, CV_8U) : inputMaskMat; + //gaussian filtering is optional src.convertTo(templateFloat, templateFloat.type()); GaussianBlur(templateFloat, templateFloat, Size(5, 5), 0, 0);//is in-place filtering slower? + Mat preMaskFloat = Mat(hd, wd, CV_32F); + preMask.convertTo(preMaskFloat, preMaskFloat.type()); + GaussianBlur(preMaskFloat, preMaskFloat, Size(5, 5), 0, 0);//is in-place filtering slower? + // Change threshold. + preMaskFloat *= (0.5/0.95); + // Rounding conversion. + preMaskFloat.convertTo(preMask, preMask.type()); + preMask.convertTo(preMaskFloat, preMaskFloat.type()); + dst.convertTo(imageFloat, imageFloat.type()); GaussianBlur(imageFloat, imageFloat, Size(5, 5), 0, 0); @@ -420,6 +434,8 @@ double cv::findTransformECC(InputArray templateImage, filter2D(imageFloat, gradientX, -1, dx); filter2D(imageFloat, gradientY, -1, dx.t()); + gradientX = gradientX.mul(preMaskFloat); + gradientY = gradientY.mul(preMaskFloat); // matrices needed for solving linear equation system for maximizing ECC Mat jacobian = Mat(hs, ws*numberOfParameters, CV_32F); @@ -449,22 +465,22 @@ double cv::findTransformECC(InputArray templateImage, warpAffine(imageFloat, imageWarped, map, imageWarped.size(), imageFlags); warpAffine(gradientX, gradientXWarped, map, gradientXWarped.size(), imageFlags); warpAffine(gradientY, gradientYWarped, map, gradientYWarped.size(), imageFlags); - warpAffine(allOnes, imageMask, map, imageMask.size(), maskFlags); + warpAffine(preMask, imageMask, map, imageMask.size(), maskFlags); } else { warpPerspective(imageFloat, imageWarped, map, imageWarped.size(), imageFlags); warpPerspective(gradientX, gradientXWarped, map, gradientXWarped.size(), imageFlags); warpPerspective(gradientY, gradientYWarped, map, gradientYWarped.size(), imageFlags); - warpPerspective(allOnes, imageMask, map, imageMask.size(), maskFlags); + warpPerspective(preMask, imageMask, map, imageMask.size(), maskFlags); } - Scalar imgMean, imgStd, tmpMean, tmpStd; meanStdDev(imageWarped, imgMean, imgStd, imageMask); meanStdDev(templateFloat, tmpMean, tmpStd, imageMask); subtract(imageWarped, imgMean, imageWarped, imageMask);//zero-mean input + templateZM = Mat::zeros(templateZM.rows, templateZM.cols, templateZM.type()); subtract(templateFloat, tmpMean, templateZM, imageMask);//zero-mean template const double tmpNorm = std::sqrt(countNonZero(imageMask)*(tmpStd.val[0])*(tmpStd.val[0])); @@ -496,6 +512,9 @@ double cv::findTransformECC(InputArray templateImage, // calculate enhanced correlation coefficiont (ECC)->rho last_rho = rho; rho = correlation/(imgNorm*tmpNorm); + if (isnan(rho)) { + CV_Error(Error::StsNoConv, "NaN encountered."); + } // project images into jacobian project_onto_jacobian_ECC( jacobian, imageWarped, imageProjection); diff --git a/modules/video/test/test_ecc.cpp b/modules/video/test/test_ecc.cpp index 6b60a181ff..675b6a61d3 100644 --- a/modules/video/test/test_ecc.cpp +++ b/modules/video/test/test_ecc.cpp @@ -394,8 +394,91 @@ void CV_ECC_Test_Homography::run(int from) ts->set_failed_test_info(cvtest::TS::OK); } +class CV_ECC_Test_Mask : public CV_ECC_BaseTest +{ +public: + CV_ECC_Test_Mask(); +protected: + void run(int); + + bool testMask(int); +}; + +CV_ECC_Test_Mask::CV_ECC_Test_Mask(){} + +bool CV_ECC_Test_Mask::testMask(int from) +{ + Mat img = imread( string(ts->get_data_path()) + "shared/fruits.png", 0); + + + if (img.empty()) + { + ts->printf( ts->LOG, "test image can not be read"); + ts->set_failed_test_info(cvtest::TS::FAIL_INVALID_TEST_DATA); + return false; + } + Mat scaledImage; + resize(img, scaledImage, Size(216, 216)); + + Mat_ testImg; + scaledImage.convertTo(testImg, testImg.type()); + + cv::RNG rng = ts->get_rng(); + + int progress=0; + + for (int k=from; kupdate_context( this, k, true ); + progress = update_progress(progress, k, ntests, 0); + + Mat translationGround = (Mat_(2,3) << 1, 0, (rng.uniform(10.f, 20.f)), + 0, 1, (rng.uniform(10.f, 20.f))); + + Mat warpedImage; + + warpAffine(testImg, warpedImage, translationGround, + Size(200,200), INTER_LINEAR + WARP_INVERSE_MAP); + + Mat mapTranslation = (Mat_(2,3) << 1, 0, 0, 0, 1, 0); + + Mat_ mask = Mat_::ones(testImg.rows, testImg.cols); + for (int i=testImg.rows*2/3; iset_failed_test_info(cvtest::TS::FAIL_INVALID_OUTPUT); + return false; + } + + if (computeRMS(mapTranslation, translationGround)>MAX_RMS_ECC){ + ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY); + ts->printf( ts->LOG, "RMS = %f", + computeRMS(mapTranslation, translationGround)); + return false; + } + + } + return true; +} + +void CV_ECC_Test_Mask::run(int from) +{ + if (!testMask(from)) + return; + + ts->set_failed_test_info(cvtest::TS::OK); +} TEST(Video_ECC_Translation, accuracy) { CV_ECC_Test_Translation test; test.safe_run();} TEST(Video_ECC_Euclidean, accuracy) { CV_ECC_Test_Euclidean test; test.safe_run(); } TEST(Video_ECC_Affine, accuracy) { CV_ECC_Test_Affine test; test.safe_run(); } TEST(Video_ECC_Homography, accuracy) { CV_ECC_Test_Homography test; test.safe_run(); } +TEST(Video_ECC_Mask, accuracy) { CV_ECC_Test_Mask test; test.safe_run(); }