From 926b7af8c9d467a1d00603ffacf44f2a056ad27b Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Fri, 18 Aug 2017 13:03:52 +0300 Subject: [PATCH 01/11] tracking: hide unused debug method in MedianFlow --- modules/tracking/src/trackerMedianFlow.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/tracking/src/trackerMedianFlow.cpp b/modules/tracking/src/trackerMedianFlow.cpp index c751aa219..db1e80557 100644 --- a/modules/tracking/src/trackerMedianFlow.cpp +++ b/modules/tracking/src/trackerMedianFlow.cpp @@ -338,7 +338,6 @@ Rect2d TrackerMedianFlowImpl::vote(const std::vector& oldPoints,const s return newRect; } - #if 0 void TrackerMedianFlowImpl::computeStatistics(std::vector& data,int size){ int binnum=10; @@ -355,7 +354,6 @@ void TrackerMedianFlowImpl::computeStatistics(std::vector& data,int size) } } #endif - void TrackerMedianFlowImpl::check_FB(const std::vector& oldImagePyr, const std::vector& newImagePyr, const std::vector& oldPoints, const std::vector& newPoints, std::vector& status){ From 255de8777b754a5385a326f761f2057060c4a9f6 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Fri, 18 Aug 2017 14:01:45 +0300 Subject: [PATCH 02/11] tracking: fix wrong conditions of OCL run --- modules/tracking/src/tldTracker.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index d83b7057a..4d7596891 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -149,8 +149,10 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { #ifdef HAVE_OPENCL - if (ocl::haveOpenCL()) + if (ocl::useOpenCL()) + { DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + } else #endif DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); @@ -221,7 +223,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL - if (ocl::haveOpenCL()) + if (ocl::useOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); else #endif @@ -230,7 +232,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) nExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL - if (ocl::haveOpenCL()) + if (ocl::useOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); else #endif From e83871228c0ffbed9c7d6039e7b93844a31cd87f Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Fri, 18 Aug 2017 16:57:20 +0300 Subject: [PATCH 03/11] TLD: fix NCC calculation --- modules/tracking/src/tldUtils.cpp | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/modules/tracking/src/tldUtils.cpp b/modules/tracking/src/tldUtils.cpp index 0792e726c..f01f30a56 100644 --- a/modules/tracking/src/tldUtils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -171,20 +171,14 @@ double NCC(const Mat_& patch1, const Mat_& patch2) CV_Assert( patch1.cols == patch2.cols ); int N = patch1.rows * patch1.cols; - int s1 = 0, s2 = 0, n1 = 0, n2 = 0, prod = 0; - for( int i = 0; i < patch1.rows; i++ ) - { - for( int j = 0; j < patch1.cols; j++ ) - { - int p1 = patch1(i, j), p2 = patch2(i, j); - s1 += p1; s2 += p2; - n1 += (p1 * p1); n2 += (p2 * p2); - prod += (p1 * p2); - } - } - double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)), sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); - double ares = (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; - return ares; + double s1 = sum(patch1)(0); + double s2 = sum(patch2)(0); + double n1 = norm(patch1, NORM_L2SQR); + double n2 = norm(patch2, NORM_L2SQR); + double prod=patch1.dot(patch2); + double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); + double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); + return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; } int getMedian(const std::vector& values, int size) From ccfd55dc7a04e7bfcf55d6d3b5c33186e636ab16 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Mon, 21 Aug 2017 17:45:56 +0300 Subject: [PATCH 04/11] tracking: enable tests for TLD --- modules/tracking/test/test_trackers.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/tracking/test/test_trackers.cpp b/modules/tracking/test/test_trackers.cpp index 2a83cbd0a..f3ddd1c41 100644 --- a/modules/tracking/test/test_trackers.cpp +++ b/modules/tracking/test/test_trackers.cpp @@ -459,7 +459,7 @@ TEST_P(DistanceAndOverlap, KCF) test.run(); } -TEST_P(DistanceAndOverlap, DISABLED_TLD) +TEST_P(DistanceAndOverlap, TLD) { TrackerTest test( TrackerTLD::create(), dataset, 60, .4f, NoTransform); test.run(); @@ -490,7 +490,7 @@ TEST_P(DistanceAndOverlap, Shifted_Data_KCF) test.run(); } -TEST_P(DistanceAndOverlap, DISABLED_Shifted_Data_TLD) +TEST_P(DistanceAndOverlap, Shifted_Data_TLD) { TrackerTest test( TrackerTLD::create(), dataset, 120, .2f, CenterShiftLeft); test.run(); @@ -521,7 +521,7 @@ TEST_P(DistanceAndOverlap, Scaled_Data_KCF) test.run(); } -TEST_P(DistanceAndOverlap, DISABLED_Scaled_Data_TLD) +TEST_P(DistanceAndOverlap, Scaled_Data_TLD) { TrackerTest test( TrackerTLD::create(), dataset, 120, .45f, Scale_1_1); test.run(); From b8588f845f651c4bd8f248ae4b3b25cd10ddc69c Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 22 Aug 2017 12:27:47 +0300 Subject: [PATCH 05/11] tracking: disable OCL in TLD (kernels are slow and broken) --- modules/tracking/src/tldTracker.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 4d7596891..e2d732729 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -149,14 +149,14 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { #ifdef HAVE_OPENCL - if (ocl::useOpenCL()) + if (false)//ocl::useOpenCL()) { - DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); } else #endif - DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - } + DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + } if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) { candidates.push_back(tmpCandid); @@ -223,7 +223,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL - if (ocl::useOpenCL()) + if (false)//ocl::useOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); else #endif @@ -232,7 +232,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) nExpert.additionalExamples(examplesForModel, examplesForEnsemble); #ifdef HAVE_OPENCL - if (ocl::useOpenCL()) + if (false)//ocl::useOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); else #endif From a6215264ed4b1278fb456b854ad1aed3e99eb325 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 29 Aug 2017 14:51:58 +0300 Subject: [PATCH 06/11] tracking: eliminate code duplication --- modules/tracking/src/tldDetector.cpp | 15 +++--- modules/tracking/src/tldUtils.cpp | 29 ----------- modules/tracking/src/tldUtils.hpp | 5 -- modules/tracking/src/trackerMedianFlow.cpp | 58 +++------------------- modules/tracking/src/tracking_utils.cpp | 23 +++++++++ modules/tracking/src/tracking_utils.hpp | 47 ++++++++++++++++++ 6 files changed, 86 insertions(+), 91 deletions(-) create mode 100644 modules/tracking/src/tracking_utils.cpp create mode 100644 modules/tracking/src/tracking_utils.hpp diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 633a60b0a..3dac8b53b 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -40,6 +40,7 @@ //M*/ #include "tldDetector.hpp" +#include "tracking_utils.hpp" #include @@ -72,12 +73,12 @@ namespace cv for (int i = 0; i < *posNum; i++) { modelSample.data = &(posExp->data[i * 225]); - splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0)); + splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } for (int i = 0; i < *negNum; i++) { modelSample.data = &(negExp->data[i * 225]); - sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); + sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } if (splus + sminus == 0.0) @@ -167,7 +168,7 @@ namespace cv for (int id = 0; id < numOfPatches; id++) { double spr = 0.0, smr = 0.0, spc = 0.0, smc = 0; - int med = getMedian((*timeStampsPositive)); + int med = tracking_internal::getMedian((*timeStampsPositive)); for (int i = 0; i < *posNum; i++) { spr = std::max(spr, 0.5 * (posNCC.at(id * 500 + i) + 1.0)); @@ -195,19 +196,19 @@ namespace cv { double splus = 0.0, sminus = 0.0; Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - int med = getMedian((*timeStampsPositive)); + int med = tracking_internal::getMedian((*timeStampsPositive)); for (int i = 0; i < *posNum; i++) { if ((int)(*timeStampsPositive)[i] <= med) { modelSample.data = &(posExp->data[i * 225]); - splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0)); + splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } } for (int i = 0; i < *negNum; i++) { modelSample.data = &(negExp->data[i * 225]); - sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); + sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } if (splus + sminus == 0.0) @@ -249,7 +250,7 @@ namespace cv Mat resNCC = devNCC.getMat(ACCESS_READ); - int med = getMedian((*timeStampsPositive)); + int med = tracking_internal::getMedian((*timeStampsPositive)); for (int i = 0; i < *posNum; i++) if ((int)(*timeStampsPositive)[i] <= med) splus = std::max(splus, 0.5 * (resNCC.at(i) +1.0)); diff --git a/modules/tracking/src/tldUtils.cpp b/modules/tracking/src/tldUtils.cpp index f01f30a56..17377c793 100644 --- a/modules/tracking/src/tldUtils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -164,35 +164,6 @@ double variance(const Mat& img) return p2 - p * p; } -//Normalized Correlation Coefficient -double NCC(const Mat_& patch1, const Mat_& patch2) -{ - CV_Assert( patch1.rows == patch2.rows ); - CV_Assert( patch1.cols == patch2.cols ); - - int N = patch1.rows * patch1.cols; - double s1 = sum(patch1)(0); - double s2 = sum(patch2)(0); - double n1 = norm(patch1, NORM_L2SQR); - double n2 = norm(patch2, NORM_L2SQR); - double prod=patch1.dot(patch2); - double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); - double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); - return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; -} - -int getMedian(const std::vector& values, int size) -{ - if( size == -1 ) - size = (int)values.size(); - std::vector copy(values.begin(), values.begin() + size); - std::sort(copy.begin(), copy.end()); - if( size % 2 == 0 ) - return (copy[size / 2 - 1] + copy[size / 2]) / 2; - else - return copy[(size - 1) / 2]; -} - //Overlap between two BB double overlap(const Rect2d& r1, const Rect2d& r2) { diff --git a/modules/tracking/src/tldUtils.hpp b/modules/tracking/src/tldUtils.hpp index e6090069a..1a606429d 100644 --- a/modules/tracking/src/tldUtils.hpp +++ b/modules/tracking/src/tldUtils.hpp @@ -46,13 +46,8 @@ namespace cv void resample(const Mat& img, const Rect2d& r2, Mat_& samples); /** Computes the variance of single given image.*/ double variance(const Mat& img); - /** Computes normalized corellation coefficient between the two patches (they should be - * of the same size).*/ - double NCC(const Mat_& patch1, const Mat_& patch2); void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res); double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep); - int getMedian(const std::vector& values, int size = -1); - } } diff --git a/modules/tracking/src/trackerMedianFlow.cpp b/modules/tracking/src/trackerMedianFlow.cpp index db1e80557..57a0e2a4b 100644 --- a/modules/tracking/src/trackerMedianFlow.cpp +++ b/modules/tracking/src/trackerMedianFlow.cpp @@ -42,6 +42,7 @@ #include "precomp.hpp" #include "opencv2/video/tracking.hpp" #include "opencv2/imgproc.hpp" +#include "tracking_utils.hpp" #include #include @@ -94,12 +95,6 @@ private: TrackerMedianFlow::Params params; }; -template -T getMedian( const std::vector& values ); - -template -T getMedianAndDoPartition( std::vector& values ); - Mat getPatch(Mat image, Size patch_size, Point2f patch_center) { Mat patch; @@ -282,7 +277,7 @@ bool TrackerMedianFlowImpl::medianFlowImpl(Mat oldImage,Mat newImage,Rect2d& old di[i]-=mDisplacement; displacements.push_back((float)sqrt(di[i].ddot(di[i]))); } - float median_displacements = getMedianAndDoPartition(displacements); + float median_displacements = tracking_internal::getMedianAndDoPartition(displacements); dprintf(("\tmedian of length of difference of displacements = %f\n", median_displacements)); if(median_displacements > params.maxMedianLengthOfDisplacementDifference){ dprintf(("\tmedian flow tracker returns false due to big median length of difference between displacements\n")); @@ -310,10 +305,10 @@ Rect2d TrackerMedianFlowImpl::vote(const std::vector& oldPoints,const s float xshift=0,yshift=0; std::vector buf_for_location(n, 0.); for(size_t i=0;i& oldPoints,const s } } - double scale=getMedianAndDoPartition(buf_for_scale); + double scale=tracking_internal::getMedianAndDoPartition(buf_for_scale); dprintf(("xshift, yshift, scale = %f %f %f\n",xshift,yshift,scale)); newRect.x=newCenter.x-scale*oldRect.width/2.0; newRect.y=newCenter.y-scale*oldRect.height/2.0; @@ -371,7 +366,7 @@ void TrackerMedianFlowImpl::check_FB(const std::vector& oldImagePyr, const for(size_t i=0;i= median); } } -template -T getMedian(const std::vector& values) -{ - std::vector copy(values); - return getMedianAndDoPartition(copy); -} - -template -T getMedianAndDoPartition(std::vector& values) -{ - size_t size = values.size(); - if(size%2==0) - { - std::nth_element(values.begin(), values.begin() + size/2-1, values.end()); - T firstMedian = values[size/2-1]; - - std::nth_element(values.begin(), values.begin() + size/2, values.end()); - T secondMedian = values[size/2]; - - return (firstMedian + secondMedian) / (T)2; - } - else - { - size_t medianIndex = (size - 1) / 2; - std::nth_element(values.begin(), values.begin() + medianIndex, values.end()); - - return values[medianIndex]; - } -} - } /* anonymous namespace */ namespace cv diff --git a/modules/tracking/src/tracking_utils.cpp b/modules/tracking/src/tracking_utils.cpp new file mode 100644 index 000000000..ee0cc2c02 --- /dev/null +++ b/modules/tracking/src/tracking_utils.cpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "tracking_utils.hpp" + +using namespace cv; + +double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) +{ + CV_Assert( patch1.rows == patch2.rows ); + CV_Assert( patch1.cols == patch2.cols ); + + int N = patch1.rows * patch1.cols; + double s1 = sum(patch1)(0); + double s2 = sum(patch2)(0); + double n1 = norm(patch1, NORM_L2SQR); + double n2 = norm(patch2, NORM_L2SQR); + double prod=patch1.dot(patch2); + double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); + double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); + return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; +} diff --git a/modules/tracking/src/tracking_utils.hpp b/modules/tracking/src/tracking_utils.hpp new file mode 100644 index 000000000..a73562242 --- /dev/null +++ b/modules/tracking/src/tracking_utils.hpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +#ifndef __OPENCV_TRACKING_UTILS_HPP__ + +#include "precomp.hpp" +#include + +namespace cv { +namespace tracking_internal +{ +/** Computes normalized corellation coefficient between the two patches (they should be +* of the same size).*/ + double computeNCC(const Mat& patch1, const Mat& patch2); + + template + T getMedianAndDoPartition(std::vector& values) + { + size_t size = values.size(); + if(size%2==0) + { + std::nth_element(values.begin(), values.begin() + size/2-1, values.end()); + T firstMedian = values[size/2-1]; + + std::nth_element(values.begin(), values.begin() + size/2, values.end()); + T secondMedian = values[size/2]; + + return (firstMedian + secondMedian) / (T)2; + } + else + { + size_t medianIndex = (size - 1) / 2; + std::nth_element(values.begin(), values.begin() + medianIndex, values.end()); + + return values[medianIndex]; + } + } + + template + T getMedian(const std::vector& values) + { + std::vector copy(values); + return getMedianAndDoPartition(copy); + } +} +} +#endif From 895a9143a6d95a0a9c61f5b6b508922e674478a7 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Wed, 30 Aug 2017 15:08:38 +0300 Subject: [PATCH 07/11] tracking: fix unitialized memory access in TLD --- modules/tracking/src/tldModel.cpp | 20 +++++++++++--------- modules/tracking/src/tldTracker.cpp | 17 ++++++++++------- modules/tracking/src/tldTracker.hpp | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 5598ca51c..7ff43e2af 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -87,9 +87,9 @@ namespace cv //Generate initial positive samples and put them to the model positiveExamples.reserve(200); - for (int i = 0; i < (int)closest.size(); i++) + for (size_t i = 0; i < closest.size(); i++) { - for (int j = 0; j < 20; j++) + for (size_t j = 0; j < 20; j++) { Point2f center; Size2f size; @@ -102,13 +102,15 @@ namespace cv resample(scaledImg, RotatedRect(center, size, angle), standardPatch); - for (int y = 0; y < standardPatch.rows; y++) - { - for (int x = 0; x < standardPatch.cols; x++) - { - standardPatch(x, y) += (uchar)rng.gaussian(5.0); - } - } + for( int y = 0; y < standardPatch.rows; y++ ) + { + uchar* patchRow = standardPatch.ptr(y); + for( int x = 0; x < standardPatch.cols; x++ ) + { + int newValue = patchRow[x] + cvRound(rng.gaussian(5.0)); + patchRow[x] = saturate_cast(newValue); + } + } #ifdef BLUR_AS_VADIM GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index e2d732729..57e451455 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -41,7 +41,6 @@ #include "tldTracker.hpp" - namespace cv { @@ -156,7 +155,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) else #endif DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - } + } if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) { candidates.push_back(tmpCandid); @@ -174,7 +173,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) } std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); - if( it == candidatesRes.end() ) + if( it == candidatesRes.end() ) //candidates are empty { data->confident = false; data->failedLastTime = true; @@ -259,6 +258,7 @@ int TrackerTLDImpl::Pexpert::additionalExamples(std::vector >& examp double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)), scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); + TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid); getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest); @@ -275,21 +275,24 @@ int TrackerTLDImpl::Pexpert::additionalExamples(std::vector >& examp size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); float angle = (float)rng.uniform(-5.0, 5.0); + resample(scaledImg, RotatedRect(center, size, angle), standardPatch); for( int y = 0; y < standardPatch.rows; y++ ) { + uchar* patchRow = standardPatch.ptr(y); for( int x = 0; x < standardPatch.cols; x++ ) { - standardPatch(x, y) += (uchar)rng.gaussian(5.0); + int newValue = patchRow[x] + cvRound(rng.gaussian(5.0)); + patchRow[x] = saturate_cast(newValue); } } -#ifdef BLUR_AS_VADIM + examplesForModel.push_back(standardPatch); + +#if defined BLUR_AS_VADIM GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); resize(blurredPatch, blurredPatch, initSize_); #else resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); #endif - resample(scaledImg, RotatedRect(center, size, angle), standardPatch); - examplesForModel.push_back(standardPatch); examplesForEnsemble.push_back(blurredPatch); } } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 5e544bbdc..069878f46 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -112,7 +112,7 @@ private: }; -#define BLUR_AS_VADIM +#undef BLUR_AS_VADIM #undef CLOSED_LOOP class TrackerTLDImpl : public TrackerTLD From a0896c7bd1e5cab86d53cb2eaecb78fdfab20aeb Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Thu, 31 Aug 2017 10:32:24 +0300 Subject: [PATCH 08/11] tracking: update tresholds in regression tests of the TLD --- modules/tracking/src/multiTracker.cpp | 2 +- modules/tracking/src/tldDetector.cpp | 4 ++-- modules/tracking/src/tldDetector.hpp | 3 ++- modules/tracking/src/tldTracker.cpp | 4 ++-- modules/tracking/src/tldUtils.cpp | 11 +++-------- modules/tracking/test/test_trackers.cpp | 6 +++--- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 33b691457..04e1c0c00 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -408,7 +408,7 @@ namespace cv ////To fix: Check the paper, probably this cause wrong learning // labPatch.isObject = srValue > tld::THETA_NN; - labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1; + labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < tld::CLASSIFIER_MARGIN; patches[k].push_back(labPatch); // diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 3dac8b53b..0c2092084 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -420,7 +420,7 @@ namespace cv ////To fix: Check the paper, probably this cause wrong learning // labPatch.isObject = srValue > THETA_NN; - labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1; + labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < CLASSIFIER_MARGIN; patches.push_back(labPatch); // @@ -539,7 +539,7 @@ namespace cv ////To fix: Check the paper, probably this cause wrong learning // labPatch.isObject = srValue > THETA_NN; - labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1; + labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < CLASSIFIER_MARGIN; patches.push_back(labPatch); // diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 4fbc4dff2..dcb460b6a 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -57,8 +57,9 @@ namespace cv const int MEASURES_PER_CLASSIFIER = 13; const int GRIDSIZE = 15; const int DOWNSCALE_MODE = cv::INTER_LINEAR; - const double THETA_NN = 0.50; + const double THETA_NN = 0.5; const double CORE_THRESHOLD = 0.5; + const double CLASSIFIER_MARGIN = 0.1; const double SCALE_STEP = 1.2; const double ENSEMBLE_THRESHOLD = 0.5; const double VARIANCE_THRESHOLD = 0.5; diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 57e451455..8bb8c8a19 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -262,9 +262,9 @@ int TrackerTLDImpl::Pexpert::additionalExamples(std::vector >& examp TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid); getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest); - for( int i = 0; i < (int)closest.size(); i++ ) + for( size_t i = 0; i < closest.size(); i++ ) { - for( int j = 0; j < 10; j++ ) + for( size_t j = 0; j < 10; j++ ) { Point2f center; Size2f size; diff --git a/modules/tracking/src/tldUtils.cpp b/modules/tracking/src/tldUtils.cpp index 17377c793..bed90bc90 100644 --- a/modules/tracking/src/tldUtils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -151,16 +151,11 @@ void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector< double variance(const Mat& img) { double p = 0, p2 = 0; - for( int i = 0; i < img.rows; i++ ) - { - for( int j = 0; j < img.cols; j++ ) - { - p += img.at(i, j); - p2 += img.at(i, j) * img.at(i, j); - } - } + p = sum(img)(0); + p2 = norm(img, NORM_L2SQR); p /= (img.cols * img.rows); p2 /= (img.cols * img.rows); + return p2 - p * p; } diff --git a/modules/tracking/test/test_trackers.cpp b/modules/tracking/test/test_trackers.cpp index f3ddd1c41..d01be0885 100644 --- a/modules/tracking/test/test_trackers.cpp +++ b/modules/tracking/test/test_trackers.cpp @@ -461,7 +461,7 @@ TEST_P(DistanceAndOverlap, KCF) TEST_P(DistanceAndOverlap, TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 60, .4f, NoTransform); + TrackerTest test( TrackerTLD::create(), dataset, 40, .45f, NoTransform); test.run(); } /***************************************************************************************/ @@ -492,7 +492,7 @@ TEST_P(DistanceAndOverlap, Shifted_Data_KCF) TEST_P(DistanceAndOverlap, Shifted_Data_TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 120, .2f, CenterShiftLeft); + TrackerTest test( TrackerTLD::create(), dataset, 30, .35f, CenterShiftLeft); test.run(); } /***************************************************************************************/ @@ -523,7 +523,7 @@ TEST_P(DistanceAndOverlap, Scaled_Data_KCF) TEST_P(DistanceAndOverlap, Scaled_Data_TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 120, .45f, Scale_1_1); + TrackerTest test( TrackerTLD::create(), dataset, 30, .45f, Scale_1_1); test.run(); } From 62939e2937f1a0627d6cbcffa2b73529d44f9c21 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Wed, 6 Sep 2017 12:34:36 +0300 Subject: [PATCH 09/11] tracking: slightly rewrite main loop in TLD --- modules/tracking/src/tldTracker.cpp | 50 ++++++++++++++--------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 8bb8c8a19..00ee55cc5 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -140,37 +140,35 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) std::vector candidates; std::vector candidatesRes; bool trackerNeedsReInit = false; - bool DETECT_FLG = false; - for( int i = 0; i < 2; i++ ) + bool DETECT_FLG = false; + + //run tracker + Rect2d tmpCandid = boundingBox; + if(!data->failedLastTime && trackerProxy->update(image, tmpCandid)) { - Rect2d tmpCandid = boundingBox; + candidates.push_back(tmpCandid); + resample(image_gray, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + } + else + trackerNeedsReInit = true; - if (i == 1) - { + //run detector + tmpCandid = boundingBox; #ifdef HAVE_OPENCL - if (false)//ocl::useOpenCL()) - { - DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - } - else + if (false)//ocl::useOpenCL()) + DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + else #endif - DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - } - if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) - { - candidates.push_back(tmpCandid); - if( i == 0 ) - resample(image_gray, tmpCandid, standardPatch); - else - resample(imageForDetector, tmpCandid, standardPatch); - candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); - } - else - { - if( i == 0 ) - trackerNeedsReInit = true; - } + DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + + if(DETECT_FLG) + { + candidates.push_back(tmpCandid); + resample(imageForDetector, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); } + std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); if( it == candidatesRes.end() ) //candidates are empty From 941865c9a12f63d5450e937bf2c774186cf8ef07 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Thu, 7 Sep 2017 14:29:52 +0300 Subject: [PATCH 10/11] tracking: reduce useless computations in TLD --- modules/tracking/src/tldDetector.cpp | 63 ++++++++++++++++++++-------- modules/tracking/src/tldDetector.hpp | 5 +++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 0c2092084..bef649c29 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -65,27 +65,58 @@ namespace cv return p; } + double TLDDetector::computeSminus(const Mat_& patch) const + { + double sminus = 0.0; + Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + for (int i = 0; i < *negNum; i++) + { + modelSample.data = &(negExp->data[i * 225]); + sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); + } + return sminus; + } + // Calculate Relative similarity of the patch (NN-Model) double TLDDetector::Sr(const Mat_& patch) const { double splus = 0.0, sminus = 0.0; - Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); for (int i = 0; i < *posNum; i++) { modelSample.data = &(posExp->data[i * 225]); splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } - for (int i = 0; i < *negNum; i++) - { - modelSample.data = &(negExp->data[i * 225]); - sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); - } + sminus = computeSminus(patch); if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); } + std::pair TLDDetector::SrAndSc(const Mat_& patch) const + { + double splusC = 0.0, sminus = 0.0, splus = 0.0; + Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + int med = tracking_internal::getMedian((*timeStampsPositive)); + for (int i = 0; i < *posNum; i++) + { + modelSample.data = &(posExp->data[i * 225]); + double s = 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0); + + if ((int)(*timeStampsPositive)[i] <= med) + splusC = std::max(splusC, s); + + splus = std::max(splus, s); + } + sminus = computeSminus(patch); + + double sr = (splus + sminus == 0.0) ? 0. : splus / (sminus + splus); + double sc = (splusC + sminus == 0.0) ? 0. : splusC / (sminus + splusC); + + return std::pair(sr, sc); + } + #ifdef HAVE_OPENCL double TLDDetector::ocl_Sr(const Mat_& patch) { @@ -205,11 +236,7 @@ namespace cv splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); } } - for (int i = 0; i < *negNum; i++) - { - modelSample.data = &(negExp->data[i * 225]); - sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0)); - } + sminus = computeSminus(patch); if (splus + sminus == 0.0) return 0.0; @@ -317,9 +344,9 @@ namespace cv resample(detectorF->resized_imgs[detectorF->ensScaleIDs[ind]], Rect2d(detectorF->ensBuffer[ind], initSizeF), detectorF->standardPatches[ind]); - - detectorF->scValues[ind] = detectorF->Sc (detectorF->standardPatches[ind]); - detectorF->srValues[ind] = detectorF->Sr (detectorF->standardPatches[ind]); + std::pair values = detectorF->SrAndSc(detectorF->standardPatches[ind]); + detectorF->scValues[ind] = values.second; + detectorF->srValues[ind] = values.first; } } @@ -413,15 +440,17 @@ namespace cv LabeledPatch labPatch; double curScale = pow(SCALE_STEP, ensScaleIDs[i]); labPatch.rect = Rect2d(ensBuffer[i].x*curScale, ensBuffer[i].y*curScale, initSize.width * curScale, initSize.height * curScale); + labPatch.Sc = scValues[i]; + //printf("max sc %f\n", labPatch.Sc); const double srValue = srValues[i]; const double scValue = scValues[i]; ////To fix: Check the paper, probably this cause wrong learning // - labPatch.isObject = srValue > THETA_NN; + labPatch.isObject = srValue > THETA_NN; labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < CLASSIFIER_MARGIN; - patches.push_back(labPatch); + patches.push_back(labPatch); // if (!labPatch.isObject) @@ -441,7 +470,7 @@ namespace cv } } - if (maxSc < 0) + if (maxSc < 0) return false; else { diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index dcb460b6a..980d59924 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -78,6 +78,7 @@ namespace cv void prepareClassifiers(int rowstep); double Sr(const Mat_& patch) const; double Sc(const Mat_& patch) const; + std::pair SrAndSc(const Mat_& patch) const; #ifdef HAVE_OPENCL double ocl_Sr(const Mat_& patch); double ocl_Sc(const Mat_& patch); @@ -102,6 +103,7 @@ namespace cv { Rect2d rect; bool isObject, shouldBeIntegrated; + double Sc; }; bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); @@ -109,6 +111,9 @@ namespace cv friend class MyMouseCallbackDEBUG; static void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } static inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); + + protected: + double computeSminus(const Mat_& patch) const; }; From 0c5a8e1fec819314ff953261d028045436a0b78d Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 26 Sep 2017 16:19:37 +0300 Subject: [PATCH 11/11] tracking: speedup NCC kernel for TLD and Median Flow --- modules/tracking/src/tracking_utils.cpp | 66 +++++++++++++++++++++---- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/modules/tracking/src/tracking_utils.cpp b/modules/tracking/src/tracking_utils.cpp index ee0cc2c02..694d87be1 100644 --- a/modules/tracking/src/tracking_utils.cpp +++ b/modules/tracking/src/tracking_utils.cpp @@ -8,16 +8,62 @@ using namespace cv; double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) { - CV_Assert( patch1.rows == patch2.rows ); - CV_Assert( patch1.cols == patch2.cols ); + CV_Assert( patch1.rows == patch2.rows, + patch1.cols == patch2.cols); int N = patch1.rows * patch1.cols; - double s1 = sum(patch1)(0); - double s2 = sum(patch2)(0); - double n1 = norm(patch1, NORM_L2SQR); - double n2 = norm(patch2, NORM_L2SQR); - double prod=patch1.dot(patch2); - double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); - double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); - return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; + + if(N <= 1000 && patch1.type() == CV_8U && patch2.type() == CV_8U) + { + unsigned s1 = 0, s2 = 0; + unsigned n1 = 0, n2 = 0; + unsigned prod = 0; + + if(patch1.isContinuous() && patch2.isContinuous()) + { + const uchar* p1Ptr = patch1.ptr(0); + const uchar* p2Ptr = patch2.ptr(0); + + for(int j = 0; j < N; j++) + { + s1 += p1Ptr[j]; + s2 += p2Ptr[j]; + n1 += p1Ptr[j]*p1Ptr[j]; + n2 += p2Ptr[j]*p2Ptr[j]; + prod += p1Ptr[j]*p2Ptr[j]; + } + } + else + { + for(int i = 0; i < patch1.rows; i++) + { + const uchar* p1Ptr = patch1.ptr(i); + const uchar* p2Ptr = patch2.ptr(i); + + for(int j = 0; j < patch1.cols; j++) + { + s1 += p1Ptr[j]; + s2 += p2Ptr[j]; + n1 += p1Ptr[j]*p1Ptr[j]; + n2 += p2Ptr[j]*p2Ptr[j]; + prod += p1Ptr[j]*p2Ptr[j]; + } + } + } + + double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); + double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); + return (sq2 == 0) ? sq1 / abs(sq1) : (prod - 1.0 * s1 * s2 / N) / sq1 / sq2; + } + else + { + double s1 = sum(patch1)(0); + double s2 = sum(patch2)(0); + double n1 = norm(patch1, NORM_L2SQR); + double n2 = norm(patch2, NORM_L2SQR); + double prod=patch1.dot(patch2); + double sq1 = sqrt(std::max(0.0, n1 - 1.0 * s1 * s1 / N)); + double sq2 = sqrt(std::max(0.0, n2 - 1.0 * s2 * s2 / N)); + return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; + } }