From 43e71214c8f1d123d5e45628685c0088d03708ea Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Tue, 19 May 2015 16:38:53 +0900 Subject: [PATCH 01/98] + add KCF Tracker, initial commit, added: tutorial, trackerKCF.cpp, modified: tracker.cpp, tracker.hpp --- ...ial] Adding new Tracker Method for dummies | 135 +++++ .../include/opencv2/tracking/tracker.hpp | 32 ++ modules/tracking/src/tracker.cpp | 1 + modules/tracking/src/trackerKCF.cpp | 472 ++++++++++++++++++ 4 files changed, 640 insertions(+) create mode 100644 modules/tracking/doc/[Tutorial] Adding new Tracker Method for dummies create mode 100644 modules/tracking/src/trackerKCF.cpp diff --git a/modules/tracking/doc/[Tutorial] Adding new Tracker Method for dummies b/modules/tracking/doc/[Tutorial] Adding new Tracker Method for dummies new file mode 100644 index 000000000..3e6159261 --- /dev/null +++ b/modules/tracking/doc/[Tutorial] Adding new Tracker Method for dummies @@ -0,0 +1,135 @@ + +/*---------------STEP 1---------------------*/ +/* modify this file +* opencv2/tracking/tracker.hpp +* and put several lines of snippet similar to +* the following: +*/ +/*------------------------------------------*/ + +class CV_EXPORTS_W TrackerKCF : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + void read( const FileNode& /*fn*/ ); + void write( FileStorage& /*fs*/ ) const; + }; + + /** @brief Constructor + @param parameters KCF parameters TrackerKCF::Params + */ + BOILERPLATE_CODE("KCF",TrackerKCF); +}; + + +/*---------------STEP 2---------------------*/ +/* modify this file +* src/tracker.cpp +* add one line in function +* Ptr Tracker::create( const String& trackerType ) +*/ +/*------------------------------------------*/ + +Ptr Tracker::create( const String& trackerType ) +{ + BOILERPLATE_CODE("MIL",TrackerMIL); + BOILERPLATE_CODE("BOOSTING",TrackerBoosting); + BOILERPLATE_CODE("MEDIANFLOW",TrackerMedianFlow); + BOILERPLATE_CODE("TLD",TrackerTLD); + BOILERPLATE_CODE("KCF",TrackerKCF); // add this line! + return Ptr(); +} + + +/*---------------STEP 3---------------------*/ +/* make a new file and paste the snippet below +* and modify it according to your needs. +* also make sure to put the LICENSE part. +* src/trackerKCF.cpp +*/ +/*------------------------------------------*/ + +/*--------------------------- +| TrackerKCFModel +|---------------------------*/ +namespace cv{ + /** + * \brief Implementation of TrackerModel for MIL algorithm + */ + class TrackerKCFModel : public TrackerModel{ + public: + TrackerKCFModel(TrackerKCF::Params /*params*/){} + ~TrackerKCFModel(){} + protected: + void modelEstimationImpl( const std::vector& responses ){} + void modelUpdateImpl(){} + }; +} /* namespace cv */ + + +/*--------------------------- +| TrackerKCF +|---------------------------*/ +namespace cv{ + + /* + * Prototype + */ + class TrackerKCFImpl : public TrackerKCF{ + public: + TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + + protected: + bool initImpl( const Mat& image, const Rect2d& boundingBox ); + bool updateImpl( const Mat& image, Rect2d& boundingBox ); + + TrackerKCF::Params params; + }; + + /* + * Constructor + */ + Ptr TrackerKCF::createTracker(const TrackerKCF::Params ¶meters){ + return Ptr(new TrackerKCFImpl(parameters)); + } + TrackerKCFImpl::TrackerKCFImpl( const TrackerKCF::Params ¶meters ) : + params( parameters ) + { + isInit = false; + } + + void TrackerKCFImpl::read( const cv::FileNode& fn ){ + params.read( fn ); + } + + void TrackerKCFImpl::write( cv::FileStorage& fs ) const{ + params.write( fs ); + } + + + bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ + model=Ptr(new TrackerKCFModel(params)); + return true; + } + bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){return true;} + + /* + * Parameters + */ + TrackerKCF::Params::Params(){ + + } + + void TrackerKCF::Params::read( const cv::FileNode& fn ){ + + } + + void TrackerKCF::Params::write( cv::FileStorage& fs ) const{ + + } + +} /* namespace cv */ diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 35191ccc3..a2b5c4684 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1189,6 +1189,38 @@ class CV_EXPORTS_W TrackerTLD : public Tracker BOILERPLATE_CODE("TLD",TrackerTLD); }; +/** @brief KCF is a novel tracking framework that explicitly decomposes the long-term tracking task into +tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector’s +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. Tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + */ +class CV_EXPORTS_W TrackerKCF : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + void read( const FileNode& /*fn*/ ); + void write( FileStorage& /*fs*/ ) const; + + double sigma; // gaussian kernel bandwidth + double lambda; // regularization + double interp_factor; // linear interpolation factor for adaptation + double output_sigma_factor; // spatial bandwidth (proportional to target) + }; + + /** @brief Constructor + @param parameters KCF parameters TrackerKCF::Params + */ + BOILERPLATE_CODE("KCF",TrackerKCF); +}; + //! @} } /* namespace cv */ diff --git a/modules/tracking/src/tracker.cpp b/modules/tracking/src/tracker.cpp index 58a46b150..cad2ffd8b 100644 --- a/modules/tracking/src/tracker.cpp +++ b/modules/tracking/src/tracker.cpp @@ -110,6 +110,7 @@ Ptr Tracker::create( const String& trackerType ) BOILERPLATE_CODE("BOOSTING",TrackerBoosting); BOILERPLATE_CODE("MEDIANFLOW",TrackerMedianFlow); BOILERPLATE_CODE("TLD",TrackerTLD); + BOILERPLATE_CODE("KCF",TrackerKCF); return Ptr(); } diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp new file mode 100644 index 000000000..a30f60646 --- /dev/null +++ b/modules/tracking/src/trackerKCF.cpp @@ -0,0 +1,472 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" +#include + +/*--------------------------- +| TrackerKCFModel +|---------------------------*/ +namespace cv{ + /** + * \brief Implementation of TrackerModel for MIL algorithm + */ + class TrackerKCFModel : public TrackerModel{ + public: + TrackerKCFModel(TrackerKCF::Params /*params*/){} + ~TrackerKCFModel(){} + protected: + void modelEstimationImpl( const std::vector& responses ){} + void modelUpdateImpl(){} + }; +} /* namespace cv */ + + +/*--------------------------- +| TrackerKCF +|---------------------------*/ +namespace cv{ + + /* + * Prototype + */ + class TrackerKCFImpl : public TrackerKCF{ + public: + TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + + protected: + /* + * basic functions and vars + */ + bool initImpl( const Mat& image, const Rect2d& boundingBox ); + bool updateImpl( const Mat& image, Rect2d& boundingBox ); + + TrackerKCF::Params params; + + /* + * KCF functions and vars + */ + void createHanningWindow(OutputArray _dst, cv::Size winSize, int type); + void inline fft2(Mat src, Mat & dest); + void inline ifft2(Mat src, Mat & dest); + void getSubWindow(Mat img, Rect roi, Mat& patch); + void denseGaussKernel(double sigma, Mat x, Mat y, Mat & k); + void calcResponse(Mat alphaf, Mat k, Mat & response); + + void shiftRows(Mat& mat); + void shiftRows(Mat& mat,int n); + void shiftCols(Mat& mat, int n); + + private: + double output_sigma; + Rect2d roi; + Mat hann; //hann window filter + + Mat y,yf; // training response and its FFT + Mat x,xf; // observation and its FFT + Mat k,kf; // dense gaussian kernel and its FFT + Mat new_alphaf, alphaf; // learning rate + Mat z, new_z; + Mat response; // detection result + + int frame; + }; + + /* + * Constructor + */ + Ptr TrackerKCF::createTracker(const TrackerKCF::Params ¶meters){ + return Ptr(new TrackerKCFImpl(parameters)); + } + TrackerKCFImpl::TrackerKCFImpl( const TrackerKCF::Params ¶meters ) : + params( parameters ) + { + isInit = false; + } + + void TrackerKCFImpl::read( const cv::FileNode& fn ){ + params.read( fn ); + } + + void TrackerKCFImpl::write( cv::FileStorage& fs ) const{ + params.write( fs ); + } + + /* + * Initialization: + * - creating hann window filter + * - ROI padding + * - creating a gaussian response for the training ground-truth + * - perform FFT to the gaussian response + */ + bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ + frame=0; + + roi = boundingBox; + + //calclulate output sigma + output_sigma=sqrt(roi.width*roi.height)*params.output_sigma_factor; + output_sigma=-0.5/(output_sigma*output_sigma); + + // add padding to the roi + roi.x-=roi.width/2; + roi.y-=roi.height/2+1; + roi.width*=2; + roi.height*=2; + + // initialize the hann window filter + createHanningWindow(hann, roi.size(), CV_64F); + + // create gaussian response + y=Mat::zeros(roi.height,roi.width,CV_64F); + for(unsigned i=0;i(i,j)=(i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1); + } + } + + y*=(double)output_sigma; + cv::exp(y,y); + + // perform fourier transfor to the gaussian response + fft2(y,yf); + + model=Ptr(new TrackerKCFModel(params)); + + // TODO: return true only if roi inside the image + return true; + } + + /* + * Main part of the KCF algorithm + */ + bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){ + double minVal, maxVal; // min-max response + Point minLoc,maxLoc; // min-max location + + Mat img; + // check the channels of the input image, grayscale is preferred + CV_Assert(image.channels() == 1 || image.channels() == 3); + if(image.channels()>1){ + cvtColor(image,img, CV_BGR2GRAY); + }else img=image; + + // extract and pre-process the patch + getSubWindow(img,roi, x); + + // detection part + if(frame>0){ + denseGaussKernel(params.sigma,x,z,k); + calcResponse(alphaf,k,response); + minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); + roi.x+=(maxLoc.x-roi.width/2+1);roi.y+=(maxLoc.y-roi.height/2+1); + + // update the bounding box + boundingBox.x=roi.x+boundingBox.width/2; + boundingBox.y=roi.y+boundingBox.height/2; + } + + // extract the patch for learning purpose + getSubWindow(img,roi, x); + + // Kernel Regularized Least-Squares, calculate alphas + denseGaussKernel(params.sigma,x,x,k); + fft2(k,kf); + kf=kf+params.lambda; + + /* TODO: optimize this element-wise division + * new_alphaf=yf./kf + * z=[(ax+bd)+i(bc-ad)]/(c^2+d^2) + */ + new_alphaf=Mat_(yf.rows, yf.cols); + std::complex temp; + for(int i=0;i(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf.at(i,j)[0],kf.at(i,j)[1])/*+complex(0.0000000001,0.0000000001)*/); + new_alphaf.at(i,j)[0]=temp.real(); + new_alphaf.at(i,j)[1]=temp.imag(); + } + } + + // update the learning model + new_z=x.clone(); + if(frame==0){ + alphaf=new_alphaf.clone(); + z=x; + }else{ + alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; + z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; + } + + frame++; + return true; + } + + + /*------------------------------------- + | implementation of the KCF functions + |-------------------------------------*/ + + /* + * hann window filter + */ + void TrackerKCFImpl::createHanningWindow(OutputArray _dst, cv::Size winSize, int type){ + CV_Assert( type == CV_32FC1 || type == CV_64FC1 ); + + _dst.create(winSize, type); + Mat dst = _dst.getMat(); + + int rows = dst.rows, cols = dst.cols; + + AutoBuffer _wc(cols); + double * const wc = (double *)_wc; + + double coeff0 = 2.0 * CV_PI / (double)(cols - 1), coeff1 = 2.0f * CV_PI / (double)(rows - 1); + for(int j = 0; j < cols; j++) + wc[j] = 0.5 * (1.0 - cos(coeff0 * j)); + + if(dst.depth() == CV_32F) + { + for(int i = 0; i < rows; i++) + { + float* dstData = dst.ptr(i); + double wr = 0.5 * (1.0 - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = (float)(wr * wc[j]); + } + } + else + { + for(int i = 0; i < rows; i++) + { + double* dstData = dst.ptr(i); + double wr = 0.5 * (1.0 - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = wr * wc[j]; + } + } + + // perform batch sqrt for SSE performance gains + //cv::sqrt(dst, dst); //matlab do not use the square rooted version + } + + /* + * simplification of fourier transoform function in opencv + */ + void inline TrackerKCFImpl::fft2(Mat src, Mat & dest){ + Mat planes[] = {Mat_(src), Mat::zeros(src.size(), CV_64F)}; + merge(planes, 2, dest); + dft(dest,dest,DFT_COMPLEX_OUTPUT); + } + + /* + * simplification of inverse fourier transoform function in opencv + */ + void inline TrackerKCFImpl::ifft2(Mat src, Mat & dest){ + idft(src,dest,DFT_SCALE+DFT_REAL_OUTPUT); + } + + /* + * obtain the patch and apply hann window filter to it + * TODO: return false if roi is outside the image, now it produce ERROR! + */ + void TrackerKCFImpl::getSubWindow(Mat img, Rect roi, Mat& patch){ + + Rect region=roi; + + // extract patch inside the image + if(roi.x<0){region.x=0;region.width+=roi.x;} + if(roi.y<0){region.y=0;region.height+=roi.y;} + if(roi.x+roi.width>img.cols)region.width=img.cols-roi.x; + if(roi.y+roi.height>img.rows)region.height=img.rows-roi.y; + if(region.width>img.cols)region.width=img.cols; + if(region.height>img.rows)region.height=img.rows; + + patch=img(region).clone(); + + // add some padding to compensate when the patch is outside image border + int addTop,addBottom, addLeft, addRight; + addTop=region.y-roi.y; + addBottom=(roi.height+roi.y>img.rows?roi.height+roi.y-img.rows:0); + addLeft=region.x-roi.x; + addRight=(roi.width+roi.x>img.cols?roi.width+roi.x-img.cols:0); + + copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); + + patch.convertTo(patch,CV_64F); + + patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 + patch=patch.mul(hann); // hann window filter + + } + + /* + * dense gauss kernel function + */ + void TrackerKCFImpl::denseGaussKernel(double sigma, Mat x, Mat y, Mat & k){ + Mat xf, yf, xyf,xy; + double normX, normY; + + fft2(x,xf); + fft2(y,yf); + normX=norm(x); + normX*=normX; + normY=norm(y); + normY*=normY; + + mulSpectrums(xf,yf,xyf,0,true); + + ifft2(xyf,xyf); + shiftRows(xyf, x.rows/2); + shiftCols(xyf,x.cols/2); + + //(xx + yy - 2 * xy) / numel(x) + xy=(normX+normY-2*xyf)/(x.rows*x.cols); + + + // TODO: check wether we really need thresholding or not + //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) + for(unsigned i=0;i(i,j)<0.0)xy.at(i,j)=0.0; + } + } + + double sig=-1.0/(sigma*sigma); + xy=sig*xy; + exp(xy,k); + + } + + /* CIRCULAR SHIT Function + * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv + */ + // circular shift one row from up to down + void TrackerKCFImpl::shiftRows(Mat& mat) { + + Mat temp; + Mat m; + int k = (mat.rows-1); + mat.row(k).copyTo(temp); + for(; k > 0 ; k-- ) { + m = mat.row(k); + mat.row(k-1).copyTo(m); + } + m = mat.row(0); + temp.copyTo(m); + + } + + // circular shift n rows from up to down if n > 0, -n rows from down to up if n < 0 + void TrackerKCFImpl::shiftRows(Mat& mat,int n) { + + if( n < 0 ) { + + n = -n; + flip(mat,mat,0); + for(int k=0; k < n;k++) { + shiftRows(mat); + } + flip(mat,mat,0); + + } else { + + for(int k=0; k < n;k++) { + shiftRows(mat); + } + } + + } + + //circular shift n columns from left to right if n > 0, -n columns from right to left if n < 0 + void TrackerKCFImpl::shiftCols(Mat& mat, int n) { + + if(n < 0){ + + n = -n; + flip(mat,mat,1); + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); + flip(mat,mat,1); + + } else { + + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); + } + } + + /* + * calculate the detection response + */ + void TrackerKCFImpl::calcResponse(Mat alphaf, Mat k, Mat & response){ + //alpha f--> 2channels ; k --> 1 channel; + Mat kf; + fft2(k,kf); + Mat spec; + mulSpectrums(alphaf,kf,spec,0,false); + ifft2(spec,response); + } + /*----------------------------------------------------------------------*/ + + /* + * Parameters + */ + TrackerKCF::Params::Params(){ + sigma=0.2; + lambda=0.01; + interp_factor=0.075; + output_sigma_factor=1.0/16.0; + } + + void TrackerKCF::Params::read( const cv::FileNode& fn ){ + + } + + void TrackerKCF::Params::write( cv::FileStorage& fs ) const{ + + } + +} /* namespace cv */ \ No newline at end of file From 89311dd406c84a2251bac59f3ef1d8548bfb20b7 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Fri, 29 May 2015 07:03:30 +0900 Subject: [PATCH 02/98] adding the resize feature --- .../include/opencv2/tracking/tracker.hpp | 13 +++--------- modules/tracking/src/trackerKCF.cpp | 20 +++++++++++++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index a2b5c4684..4a8d6dad6 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1189,16 +1189,7 @@ class CV_EXPORTS_W TrackerTLD : public Tracker BOILERPLATE_CODE("TLD",TrackerTLD); }; -/** @brief KCF is a novel tracking framework that explicitly decomposes the long-term tracking task into -tracking, learning and detection. - -The tracker follows the object from frame to frame. The detector localizes all appearances that -have been observed so far and corrects the tracker if necessary. The learning estimates detector’s -errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . - -The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this -implementation, following authors. Tracker is supposed to be able to handle rapid motions, partial -occlusions, object absence etc. +/** @brief KCF is a novel tracking framework that utilize properties of circulant matrix to enhance the processing speed. */ class CV_EXPORTS_W TrackerKCF : public Tracker { @@ -1213,6 +1204,8 @@ class CV_EXPORTS_W TrackerKCF : public Tracker double lambda; // regularization double interp_factor; // linear interpolation factor for adaptation double output_sigma_factor; // spatial bandwidth (proportional to target) + + bool resize; // activate the resize feature to improves the processing speed }; /** @brief Constructor diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index a30f60646..e9f3d0096 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -109,6 +109,8 @@ namespace cv{ Mat z, new_z; Mat response; // detection result + bool resizeImage; // resize the image whenever needed and the patch size is large + int frame; }; @@ -122,6 +124,7 @@ namespace cv{ params( parameters ) { isInit = false; + resizeImage = false; } void TrackerKCFImpl::read( const cv::FileNode& fn ){ @@ -148,6 +151,15 @@ namespace cv{ output_sigma=sqrt(roi.width*roi.height)*params.output_sigma_factor; output_sigma=-0.5/(output_sigma*output_sigma); + //resize the ROI whenever needed + if(params.resize && roi.width*roi.height>80*80){ + resizeImage=true; + roi.x/=2.0; + roi.y/=2.0; + roi.width/=2.0; + roi.height/=2.0; + } + // add padding to the roi roi.x-=roi.width/2; roi.y-=roi.height/2+1; @@ -191,6 +203,9 @@ namespace cv{ cvtColor(image,img, CV_BGR2GRAY); }else img=image; + // resize the image whenever needed + if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2)); + // extract and pre-process the patch getSubWindow(img,roi, x); @@ -202,8 +217,8 @@ namespace cv{ roi.x+=(maxLoc.x-roi.width/2+1);roi.y+=(maxLoc.y-roi.height/2+1); // update the bounding box - boundingBox.x=roi.x+boundingBox.width/2; - boundingBox.y=roi.y+boundingBox.height/2; + boundingBox.x=(resizeImage?roi.x*2:roi.x)+boundingBox.width/2; + boundingBox.y=(resizeImage?roi.y*2:roi.y)+boundingBox.height/2; } // extract the patch for learning purpose @@ -459,6 +474,7 @@ namespace cv{ lambda=0.01; interp_factor=0.075; output_sigma_factor=1.0/16.0; + resize=true; } void TrackerKCF::Params::read( const cv::FileNode& fn ){ From 816714841db139fe83ed0d0383fcb93680ba5a09 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Fri, 29 May 2015 07:13:36 +0900 Subject: [PATCH 03/98] References for KCF tracker and KCF-CN tracker --- modules/tracking/doc/tracking.bib | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modules/tracking/doc/tracking.bib b/modules/tracking/doc/tracking.bib index 95d50506f..a8dc9801a 100644 --- a/modules/tracking/doc/tracking.bib +++ b/modules/tracking/doc/tracking.bib @@ -67,3 +67,29 @@ year={2013}, organization={IEEE} } + +@article{KCF, + title = {High-Speed Tracking with Kernelized Correlation Filters}, + journal = {Pattern Analysis and Machine Intelligence, IEEE Transactions on}, + author = {Henriques, J. F. and Caseiro, R. and Martins, P. and Batista, J.}, + year = {2015}, + doi = {10.1109/TPAMI.2014.2345390}, +} + +@inproceedings{KCF_ECCV, + title = {Exploiting the Circulant Structure of Tracking-by-detection with Kernels}, + author = {Henriques, J. F. and Caseiro, R. and Martins, P. and Batista, J.}, + booktitle = {proceedings of the European Conference on Computer Vision}, + year = {2012}, +} + +@INPROCEEDINGS{KCF_CN, + author={Danelljan, M. and Khan, F.S. and Felsberg, M. and van de Weijer, J.}, + booktitle={Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on}, + title={Adaptive Color Attributes for Real-Time Visual Tracking}, + year={2014}, + month={June}, + pages={1090-1097}, + keywords={computer vision;feature extraction;image colour analysis;image representation;image sequences;adaptive color attributes;benchmark color sequences;color features;color representations;computer vision;image description;real-time visual tracking;tracking-by-detection framework;Color;Computational modeling;Covariance matrices;Image color analysis;Kernel;Target tracking;Visualization;Adaptive Dimensionality Reduction;Appearance Model;Color Features;Visual Tracking}, + doi={10.1109/CVPR.2014.143},} + \ No newline at end of file From 5d772dae42c97582c3fe71858809f0a4dd1a1bdf Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Sat, 30 May 2015 11:34:28 +0900 Subject: [PATCH 04/98] Unified the formatting --- modules/tracking/doc/tracking.bib | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/tracking/doc/tracking.bib b/modules/tracking/doc/tracking.bib index a8dc9801a..3f6d3be55 100644 --- a/modules/tracking/doc/tracking.bib +++ b/modules/tracking/doc/tracking.bib @@ -84,12 +84,12 @@ } @INPROCEEDINGS{KCF_CN, - author={Danelljan, M. and Khan, F.S. and Felsberg, M. and van de Weijer, J.}, - booktitle={Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on}, - title={Adaptive Color Attributes for Real-Time Visual Tracking}, - year={2014}, - month={June}, - pages={1090-1097}, - keywords={computer vision;feature extraction;image colour analysis;image representation;image sequences;adaptive color attributes;benchmark color sequences;color features;color representations;computer vision;image description;real-time visual tracking;tracking-by-detection framework;Color;Computational modeling;Covariance matrices;Image color analysis;Kernel;Target tracking;Visualization;Adaptive Dimensionality Reduction;Appearance Model;Color Features;Visual Tracking}, - doi={10.1109/CVPR.2014.143},} - \ No newline at end of file + author={Danelljan, M. and Khan, F.S. and Felsberg, M. and van de Weijer, J.}, + booktitle={Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on}, + title={Adaptive Color Attributes for Real-Time Visual Tracking}, + year={2014}, + month={June}, + pages={1090-1097}, + keywords={computer vision;feature extraction;image colour analysis;image representation;image sequences;adaptive color attributes;benchmark color sequences;color features;color representations;computer vision;image description;real-time visual tracking;tracking-by-detection framework;Color;Computational modeling;Covariance matrices;Image color analysis;Kernel;Target tracking;Visualization;Adaptive Dimensionality Reduction;Appearance Model;Color Features;Visual Tracking}, + doi={10.1109/CVPR.2014.143}, +} From 5027710dedc1b251505fb03608a45a3625064d92 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Sat, 30 May 2015 11:41:18 +0900 Subject: [PATCH 05/98] Fixed: ROI extraction when the given ROI is out of image; made the max_patch_size to be adjustable; add the CN feature extraction method --- .../include/opencv2/tracking/tracker.hpp | 10 +- modules/tracking/src/featureColorName.cpp | 32852 ++++++++++++++++ modules/tracking/src/trackerKCF.cpp | 57 +- 3 files changed, 32903 insertions(+), 16 deletions(-) create mode 100644 modules/tracking/src/featureColorName.cpp diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 4a8d6dad6..3e8280157 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1190,6 +1190,10 @@ class CV_EXPORTS_W TrackerTLD : public Tracker }; /** @brief KCF is a novel tracking framework that utilize properties of circulant matrix to enhance the processing speed. + * This tracking method is implementation of @cite KCF_ECCV which is extended to KFC with color-names features (@cite KCF_CN). + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * . */ class CV_EXPORTS_W TrackerKCF : public Tracker { @@ -1203,9 +1207,9 @@ class CV_EXPORTS_W TrackerKCF : public Tracker double sigma; // gaussian kernel bandwidth double lambda; // regularization double interp_factor; // linear interpolation factor for adaptation - double output_sigma_factor; // spatial bandwidth (proportional to target) - - bool resize; // activate the resize feature to improves the processing speed + double output_sigma_factor; // spatial bandwidth (proportional to target) + bool resize; // activate the resize feature to improve the processing speed + int max_patch_size; // threshold for the ROI size }; /** @brief Constructor diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp new file mode 100644 index 000000000..c64d4e4f7 --- /dev/null +++ b/modules/tracking/src/featureColorName.cpp @@ -0,0 +1,32852 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" +#include + +namespace cv{ + class ColorName{ + private: + #define nrows 32768 + #define ncolumns 10 + protected: + double cn[nrows][ncolumns]; + public: + ColorName(); + ~ColorName(){/*free(cn);*/}; + }; + + ColorName::ColorName(){ + + // try{ + // cn = (double **)malloc(nrows * sizeof(double *)); + // } + // catch(...){ + // std::cout<<"out of memory"< /*--------------------------- @@ -54,7 +55,7 @@ namespace cv{ TrackerKCFModel(TrackerKCF::Params /*params*/){} ~TrackerKCFModel(){} protected: - void modelEstimationImpl( const std::vector& responses ){} + void modelEstimationImpl( const std::vector& /*responses*/ ){} void modelUpdateImpl(){} }; } /* namespace cv */ @@ -68,7 +69,7 @@ namespace cv{ /* * Prototype */ - class TrackerKCFImpl : public TrackerKCF{ + class TrackerKCFImpl : public TrackerKCF, public ColorName{ public: TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); void read( const FileNode& fn ); @@ -89,7 +90,8 @@ namespace cv{ void createHanningWindow(OutputArray _dst, cv::Size winSize, int type); void inline fft2(Mat src, Mat & dest); void inline ifft2(Mat src, Mat & dest); - void getSubWindow(Mat img, Rect roi, Mat& patch); + bool getSubWindow(Mat img, Rect roi, Mat& patch); + void extractCN(Mat _patch, Mat & cnFeatures); void denseGaussKernel(double sigma, Mat x, Mat y, Mat & k); void calcResponse(Mat alphaf, Mat k, Mat & response); @@ -144,7 +146,6 @@ namespace cv{ */ bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ frame=0; - roi = boundingBox; //calclulate output sigma @@ -152,7 +153,7 @@ namespace cv{ output_sigma=-0.5/(output_sigma*output_sigma); //resize the ROI whenever needed - if(params.resize && roi.width*roi.height>80*80){ + if(params.resize && roi.width*roi.height>params.max_patch_size){ resizeImage=true; roi.x/=2.0; roi.y/=2.0; @@ -195,7 +196,7 @@ namespace cv{ bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){ double minVal, maxVal; // min-max response Point minLoc,maxLoc; // min-max location - + Mat img; // check the channels of the input image, grayscale is preferred CV_Assert(image.channels() == 1 || image.channels() == 3); @@ -207,7 +208,7 @@ namespace cv{ if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2)); // extract and pre-process the patch - getSubWindow(img,roi, x); + if(!getSubWindow(img,roi, x))return false; // detection part if(frame>0){ @@ -222,7 +223,7 @@ namespace cv{ } // extract the patch for learning purpose - getSubWindow(img,roi, x); + if(!getSubWindow(img,roi, x))return false; // Kernel Regularized Least-Squares, calculate alphas denseGaussKernel(params.sigma,x,x,k); @@ -323,12 +324,18 @@ namespace cv{ /* * obtain the patch and apply hann window filter to it - * TODO: return false if roi is outside the image, now it produce ERROR! */ - void TrackerKCFImpl::getSubWindow(Mat img, Rect roi, Mat& patch){ + bool TrackerKCFImpl::getSubWindow(Mat img, Rect roi, Mat& patch){ Rect region=roi; + // return false if roi is outside the image + if((roi.x+roi.width<0) + ||(roi.y+roi.height<0) + ||(roi.x>=img.cols) + ||(roi.y>=img.rows) + )return false; + // extract patch inside the image if(roi.x<0){region.x=0;region.width+=roi.x;} if(roi.y<0){region.y=0;region.height+=roi.y;} @@ -352,7 +359,30 @@ namespace cv{ patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 patch=patch.mul(hann); // hann window filter - + + return true; + + } + + /* Convert BGR to ColorNames + */ + void TrackerKCFImpl::extractCN(Mat _patch, Mat & cnFeatures){ + Vec3b & pixel = _patch.at(0,0); + unsigned index; + + cnFeatures = Mat::zeros(roi.height,roi.width,CV_64FC(10)); + + for(int i=0;i<_patch.rows;i++){ + for(int j=0;j<_patch.cols;j++){ + pixel=_patch.at(i,j); + index=floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8); + + //copy the values + for(int k=0;k<10;k++){ + cnFeatures.at >(i,j)=cn[index][k]; + } + } + } } /* @@ -381,8 +411,8 @@ namespace cv{ // TODO: check wether we really need thresholding or not //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) - for(unsigned i=0;i(i,j)<0.0)xy.at(i,j)=0.0; } } @@ -475,6 +505,7 @@ namespace cv{ interp_factor=0.075; output_sigma_factor=1.0/16.0; resize=true; + max_patch_size=80*80; } void TrackerKCF::Params::read( const cv::FileNode& fn ){ From b214736e190b0da006bad4e276b0d88e17a1be31 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Mon, 1 Jun 2015 08:26:49 +0900 Subject: [PATCH 06/98] Removing all shadowing variables, make functions to const, make the table of color-names become static --- modules/tracking/src/featureColorName.cpp | 36 ++----- modules/tracking/src/trackerKCF.cpp | 124 +++++++++++----------- 2 files changed, 68 insertions(+), 92 deletions(-) diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index c64d4e4f7..b00fdbfa8 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -47,32 +47,14 @@ namespace cv{ private: #define nrows 32768 #define ncolumns 10 - protected: - double cn[nrows][ncolumns]; public: - ColorName(); - ~ColorName(){/*free(cn);*/}; + ColorName(){}; + ~ColorName(){}; + + static double cn[nrows][ncolumns]; }; - - ColorName::ColorName(){ - - // try{ - // cn = (double **)malloc(nrows * sizeof(double *)); - // } - // catch(...){ - // std::cout<<"out of memory"<(src), Mat::zeros(src.size(), CV_64F)}; merge(planes, 2, dest); dft(dest,dest,DFT_COMPLEX_OUTPUT); @@ -318,29 +318,29 @@ namespace cv{ /* * simplification of inverse fourier transoform function in opencv */ - void inline TrackerKCFImpl::ifft2(Mat src, Mat & dest){ + void inline TrackerKCFImpl::ifft2(const Mat src, Mat & dest)const { idft(src,dest,DFT_SCALE+DFT_REAL_OUTPUT); } /* * obtain the patch and apply hann window filter to it */ - bool TrackerKCFImpl::getSubWindow(Mat img, Rect roi, Mat& patch){ + bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& patch) const{ - Rect region=roi; + Rect region=_roi; // return false if roi is outside the image - if((roi.x+roi.width<0) - ||(roi.y+roi.height<0) - ||(roi.x>=img.cols) - ||(roi.y>=img.rows) + if((_roi.x+_roi.width<0) + ||(_roi.y+_roi.height<0) + ||(_roi.x>=img.cols) + ||(_roi.y>=img.rows) )return false; // extract patch inside the image - if(roi.x<0){region.x=0;region.width+=roi.x;} - if(roi.y<0){region.y=0;region.height+=roi.y;} - if(roi.x+roi.width>img.cols)region.width=img.cols-roi.x; - if(roi.y+roi.height>img.rows)region.height=img.rows-roi.y; + if(_roi.x<0){region.x=0;region.width+=_roi.x;} + if(_roi.y<0){region.y=0;region.height+=_roi.y;} + if(_roi.x+_roi.width>img.cols)region.width=img.cols-_roi.x; + if(_roi.y+_roi.height>img.rows)region.height=img.rows-_roi.y; if(region.width>img.cols)region.width=img.cols; if(region.height>img.rows)region.height=img.rows; @@ -348,10 +348,10 @@ namespace cv{ // add some padding to compensate when the patch is outside image border int addTop,addBottom, addLeft, addRight; - addTop=region.y-roi.y; - addBottom=(roi.height+roi.y>img.rows?roi.height+roi.y-img.rows:0); - addLeft=region.x-roi.x; - addRight=(roi.width+roi.x>img.cols?roi.width+roi.x-img.cols:0); + addTop=region.y-_roi.y; + addBottom=(_roi.height+_roi.y>img.rows?_roi.height+_roi.y-img.rows:0); + addLeft=region.x-_roi.x; + addRight=(_roi.width+_roi.x>img.cols?_roi.width+_roi.x-img.cols:0); copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); @@ -366,7 +366,7 @@ namespace cv{ /* Convert BGR to ColorNames */ - void TrackerKCFImpl::extractCN(Mat _patch, Mat & cnFeatures){ + void TrackerKCFImpl::extractCN(Mat _patch, Mat & cnFeatures) const { Vec3b & pixel = _patch.at(0,0); unsigned index; @@ -378,8 +378,8 @@ namespace cv{ index=floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8); //copy the values - for(int k=0;k<10;k++){ - cnFeatures.at >(i,j)=cn[index][k]; + for(int _k=0;_k<10;_k++){ + cnFeatures.at >(i,j)[_k]=cn[index][_k]; } } } @@ -388,25 +388,25 @@ namespace cv{ /* * dense gauss kernel function */ - void TrackerKCFImpl::denseGaussKernel(double sigma, Mat x, Mat y, Mat & k){ - Mat xf, yf, xyf,xy; + void TrackerKCFImpl::denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k)const{ + Mat _xf, _yf, xyf,xy; double normX, normY; - fft2(x,xf); - fft2(y,yf); - normX=norm(x); + fft2(_x,_xf); + fft2(_y,_yf); + normX=norm(_x); normX*=normX; - normY=norm(y); + normY=norm(_y); normY*=normY; - mulSpectrums(xf,yf,xyf,0,true); + mulSpectrums(_xf,_yf,xyf,0,true); ifft2(xyf,xyf); - shiftRows(xyf, x.rows/2); - shiftCols(xyf,x.cols/2); + shiftRows(xyf, _x.rows/2); + shiftCols(xyf,_x.cols/2); //(xx + yy - 2 * xy) / numel(x) - xy=(normX+normY-2*xyf)/(x.rows*x.cols); + xy=(normX+normY-2*xyf)/(_x.rows*_x.cols); // TODO: check wether we really need thresholding or not @@ -419,7 +419,7 @@ namespace cv{ double sig=-1.0/(sigma*sigma); xy=sig*xy; - exp(xy,k); + exp(xy,_k); } @@ -427,15 +427,15 @@ namespace cv{ * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv */ // circular shift one row from up to down - void TrackerKCFImpl::shiftRows(Mat& mat) { + void TrackerKCFImpl::shiftRows(Mat& mat) const { Mat temp; Mat m; - int k = (mat.rows-1); - mat.row(k).copyTo(temp); - for(; k > 0 ; k-- ) { - m = mat.row(k); - mat.row(k-1).copyTo(m); + int _k = (mat.rows-1); + mat.row(_k).copyTo(temp); + for(; _k > 0 ; _k-- ) { + m = mat.row(_k); + mat.row(_k-1).copyTo(m); } m = mat.row(0); temp.copyTo(m); @@ -443,20 +443,20 @@ namespace cv{ } // circular shift n rows from up to down if n > 0, -n rows from down to up if n < 0 - void TrackerKCFImpl::shiftRows(Mat& mat,int n) { + void TrackerKCFImpl::shiftRows(Mat& mat, int n) const { if( n < 0 ) { n = -n; flip(mat,mat,0); - for(int k=0; k < n;k++) { + for(int _k=0; _k < n;_k++) { shiftRows(mat); } flip(mat,mat,0); } else { - for(int k=0; k < n;k++) { + for(int _k=0; _k < n;_k++) { shiftRows(mat); } } @@ -464,7 +464,7 @@ namespace cv{ } //circular shift n columns from left to right if n > 0, -n columns from right to left if n < 0 - void TrackerKCFImpl::shiftCols(Mat& mat, int n) { + void TrackerKCFImpl::shiftCols(Mat& mat, int n) const { if(n < 0){ @@ -486,13 +486,13 @@ namespace cv{ /* * calculate the detection response */ - void TrackerKCFImpl::calcResponse(Mat alphaf, Mat k, Mat & response){ + void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _k, Mat & _response)const { //alpha f--> 2channels ; k --> 1 channel; - Mat kf; - fft2(k,kf); + Mat _kf; + fft2(_k,_kf); Mat spec; - mulSpectrums(alphaf,kf,spec,0,false); - ifft2(spec,response); + mulSpectrums(_alphaf,_kf,spec,0,false); + ifft2(spec,_response); } /*----------------------------------------------------------------------*/ @@ -508,11 +508,11 @@ namespace cv{ max_patch_size=80*80; } - void TrackerKCF::Params::read( const cv::FileNode& fn ){ + void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){ } - void TrackerKCF::Params::write( cv::FileStorage& fs ) const{ + void TrackerKCF::Params::write( cv::FileStorage& /*fs*/ ) const{ } From e746608599acfe68594ee8d0837adb633cce27ca Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Mon, 1 Jun 2015 16:57:53 +0900 Subject: [PATCH 07/98] change the color-names table to const --- modules/tracking/src/featureColorName.cpp | 4 ++-- modules/tracking/src/trackerKCF.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index b00fdbfa8..7ab3244a2 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -51,10 +51,10 @@ namespace cv{ ColorName(){}; ~ColorName(){}; - static double cn[nrows][ncolumns]; + const static double cn[nrows][ncolumns]; }; - double ColorName::cn[nrows][ncolumns]={ + const double ColorName::cn[nrows][ncolumns]={ {0.45975,0.014802,0.044289,-0.028193,0.001151,-0.0050145,0.34522,0.018362,0.23994,0.1689}, {0.47157,0.021424,0.041444,-0.030215,0.0019002,-0.0029264,0.32875,0.0082059,0.2502,0.17007}, {0.47098,0.042624,0.025014,-0.033501,0.0028958,-0.001415,0.29519,-0.0072627,0.26919,0.16947}, diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index bcfe44d8f..08358defa 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -79,7 +79,7 @@ namespace cv{ /* * basic functions and vars */ - bool initImpl( const Mat& image, const Rect2d& boundingBox ); + bool initImpl( const Mat& /*image*/, const Rect2d& boundingBox ); bool updateImpl( const Mat& image, Rect2d& boundingBox ); TrackerKCF::Params params; @@ -144,7 +144,7 @@ namespace cv{ * - creating a gaussian response for the training ground-truth * - perform FFT to the gaussian response */ - bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ + bool TrackerKCFImpl::initImpl( const Mat& /*image*/, const Rect2d& boundingBox ){ frame=0; roi = boundingBox; From efc0322ec88c388fbb187fe344c9955a96d07b17 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Tue, 2 Jun 2015 14:39:46 +0900 Subject: [PATCH 08/98] Add a framework for choosing the descriptor --- .../include/opencv2/tracking/tracker.hpp | 4 ++- modules/tracking/src/trackerKCF.cpp | 30 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 3e8280157..2435b6510 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1198,6 +1198,7 @@ class CV_EXPORTS_W TrackerTLD : public Tracker class CV_EXPORTS_W TrackerKCF : public Tracker { public: + enum MODE {GRAY, CN, CN2}; struct CV_EXPORTS Params { Params(); @@ -1209,7 +1210,8 @@ class CV_EXPORTS_W TrackerKCF : public Tracker double interp_factor; // linear interpolation factor for adaptation double output_sigma_factor; // spatial bandwidth (proportional to target) bool resize; // activate the resize feature to improve the processing speed - int max_patch_size; // threshold for the ROI size + int max_patch_size; // threshold for the ROI size + MODE descriptor; // descriptor type }; /** @brief Constructor diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 08358defa..0fdc1661f 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -169,6 +169,10 @@ namespace cv{ // initialize the hann window filter createHanningWindow(hann, roi.size(), CV_64F); + if(params.descriptor==CN){ + Mat layers[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; + merge(layers, 10, hann); + } // create gaussian response y=Mat::zeros(roi.height,roi.width,CV_64F); @@ -197,12 +201,9 @@ namespace cv{ double minVal, maxVal; // min-max response Point minLoc,maxLoc; // min-max location - Mat img; + Mat img=image.clone(); // check the channels of the input image, grayscale is preferred CV_Assert(image.channels() == 1 || image.channels() == 3); - if(image.channels()>1){ - cvtColor(image,img, CV_BGR2GRAY); - }else img=image; // resize the image whenever needed if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2)); @@ -355,6 +356,20 @@ namespace cv{ copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); + // extract the desired descriptors + switch(params.descriptor){ + case GRAY: + if(img.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); + break; + case CN: + CV_Assert(img.channels() == 3); + extractCN(patch,patch); + break; + default: + if(patch.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); + break; + } + patch.convertTo(patch,CV_64F); patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 @@ -370,7 +385,7 @@ namespace cv{ Vec3b & pixel = _patch.at(0,0); unsigned index; - cnFeatures = Mat::zeros(roi.height,roi.width,CV_64FC(10)); + Mat temp = Mat::zeros(_patch.rows,_patch.cols,CV_64FC(10)); for(int i=0;i<_patch.rows;i++){ for(int j=0;j<_patch.cols;j++){ @@ -379,10 +394,12 @@ namespace cv{ //copy the values for(int _k=0;_k<10;_k++){ - cnFeatures.at >(i,j)[_k]=cn[index][_k]; + temp.at >(i,j)[_k]=cn[index][_k]; } } } + + cnFeatures=temp.clone(); } /* @@ -506,6 +523,7 @@ namespace cv{ output_sigma_factor=1.0/16.0; resize=true; max_patch_size=80*80; + descriptor=GRAY; } void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){ From 26962e86b45ee3e4269e1b0fb872672d226b6270 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 3 Jun 2015 14:53:20 +0900 Subject: [PATCH 09/98] Added error message for descriptor other than GRAY --- modules/tracking/src/trackerKCF.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 0fdc1661f..5d332188e 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -173,6 +173,7 @@ namespace cv{ Mat layers[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; merge(layers, 10, hann); } + if(params.descriptor != GRAY){printf("The choosen descriptor mode is not available! Please use GRAY descriptor, other descriptors will be avaiable soon.\n");return false;}//temporary, will be updated soon // create gaussian response y=Mat::zeros(roi.height,roi.width,CV_64F); From b2db444907b1c656cb84b1ece870ab346d2e756b Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 3 Jun 2015 14:56:34 +0900 Subject: [PATCH 10/98] Added new line at end of file --- modules/tracking/src/trackerKCF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 5d332188e..3aeb653a1 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -535,4 +535,4 @@ namespace cv{ } -} /* namespace cv */ \ No newline at end of file +} /* namespace cv */ From f4ba9a8baab631c1349621117d4e8c598ac8b329 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 3 Jun 2015 16:33:51 +0900 Subject: [PATCH 11/98] Fixing the ColorNames table initialization --- modules/tracking/src/featureColorName.cpp | 13 +------------ modules/tracking/src/trackerKCF.cpp | 4 ++-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index 7ab3244a2..8f0a42aa7 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -43,18 +43,7 @@ #include namespace cv{ - class ColorName{ - private: - #define nrows 32768 - #define ncolumns 10 - public: - ColorName(){}; - ~ColorName(){}; - - const static double cn[nrows][ncolumns]; - }; - - const double ColorName::cn[nrows][ncolumns]={ + const static double ColorNames[32768][10]={ {0.45975,0.014802,0.044289,-0.028193,0.001151,-0.0050145,0.34522,0.018362,0.23994,0.1689}, {0.47157,0.021424,0.041444,-0.030215,0.0019002,-0.0029264,0.32875,0.0082059,0.2502,0.17007}, {0.47098,0.042624,0.025014,-0.033501,0.0028958,-0.001415,0.29519,-0.0072627,0.26919,0.16947}, diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 3aeb653a1..083651982 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -69,7 +69,7 @@ namespace cv{ /* * Prototype */ - class TrackerKCFImpl : public TrackerKCF, public ColorName { + class TrackerKCFImpl : public TrackerKCF { public: TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); void read( const FileNode& /*fn*/ ); @@ -395,7 +395,7 @@ namespace cv{ //copy the values for(int _k=0;_k<10;_k++){ - temp.at >(i,j)[_k]=cn[index][_k]; + temp.at >(i,j)[_k]=ColorNames[index][_k]; } } } From df9adff57719fcb30aba123f370da83f993991cc Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 3 Jun 2015 16:59:56 +0900 Subject: [PATCH 12/98] Fixed warning: conversion from double to int --- modules/tracking/src/trackerKCF.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 083651982..c76c15a50 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -176,7 +176,7 @@ namespace cv{ if(params.descriptor != GRAY){printf("The choosen descriptor mode is not available! Please use GRAY descriptor, other descriptors will be avaiable soon.\n");return false;}//temporary, will be updated soon // create gaussian response - y=Mat::zeros(roi.height,roi.width,CV_64F); + y=Mat::zeros((int)roi.height,(int)roi.width,CV_64F); for(unsigned i=0;i(i,j)=(i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1); @@ -391,7 +391,7 @@ namespace cv{ for(int i=0;i<_patch.rows;i++){ for(int j=0;j<_patch.cols;j++){ pixel=_patch.at(i,j); - index=floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8); + index=(unsigned)(floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8)); //copy the values for(int _k=0;_k<10;_k++){ From 612b7f26b50af62043b270b8fa772913fbf359b7 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 7 Jun 2015 22:30:00 +0900 Subject: [PATCH 13/98] TLD Fixes & Optimizations 1. TLD now have module structure 2. Made some small code optimizations 3. Fixed Ensemble Classifier according to the original paper - 10 randomized ferns 4. Added comments to most of the functions and methods --- modules/tracking/include/opencv2/tracking.hpp | 3 +- .../include/opencv2/tracking/tldDataset.hpp | 15 + modules/tracking/src/tldDataset.cpp | 101 ++ modules/tracking/src/tldDetector.cpp | 195 ++++ modules/tracking/src/tldDetector.hpp | 61 ++ .../tracking/src/tldEnsembleClassifier.cpp | 158 +++ .../tracking/src/tldEnsembleClassifier.hpp | 29 + modules/tracking/src/tldModel.cpp | 236 +++++ modules/tracking/src/tldModel.hpp | 50 + modules/tracking/src/tldTracker.cpp | 299 ++++++ modules/tracking/src/tldTracker.hpp | 177 ++++ .../src/{tld_utils.cpp => tldUtils.cpp} | 169 +--- modules/tracking/src/tldUtils.hpp | 62 ++ modules/tracking/src/tld_tracker.cpp | 951 ------------------ modules/tracking/src/tld_tracker.hpp | 125 --- 15 files changed, 1399 insertions(+), 1232 deletions(-) create mode 100644 modules/tracking/include/opencv2/tracking/tldDataset.hpp create mode 100644 modules/tracking/src/tldDataset.cpp create mode 100644 modules/tracking/src/tldDetector.cpp create mode 100644 modules/tracking/src/tldDetector.hpp create mode 100644 modules/tracking/src/tldEnsembleClassifier.cpp create mode 100644 modules/tracking/src/tldEnsembleClassifier.hpp create mode 100644 modules/tracking/src/tldModel.cpp create mode 100644 modules/tracking/src/tldModel.hpp create mode 100644 modules/tracking/src/tldTracker.cpp create mode 100644 modules/tracking/src/tldTracker.hpp rename modules/tracking/src/{tld_utils.cpp => tldUtils.cpp} (61%) create mode 100644 modules/tracking/src/tldUtils.hpp delete mode 100644 modules/tracking/src/tld_tracker.cpp delete mode 100644 modules/tracking/src/tld_tracker.hpp diff --git a/modules/tracking/include/opencv2/tracking.hpp b/modules/tracking/include/opencv2/tracking.hpp index 38b60ba74..1ab2a1700 100644 --- a/modules/tracking/include/opencv2/tracking.hpp +++ b/modules/tracking/include/opencv2/tracking.hpp @@ -306,6 +306,7 @@ The first argument is the name of the tracker and the second is a video source. */ -#include "opencv2/tracking/tracker.hpp" +#include +#include #endif //__OPENCV_TRACKING_LENLEN diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp new file mode 100644 index 000000000..0505562b1 --- /dev/null +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -0,0 +1,15 @@ +#ifndef OPENCV_TLD_DATASET +#define OPENCV_TLD_DATASET + +#include "opencv2/highgui.hpp" + +namespace cv +{ + namespace tld + { + CV_EXPORTS cv::Rect2d tld_InitDataset(int datasetInd, char* rootPath = "TLD_dataset"); + CV_EXPORTS cv::Mat tld_getNextDatasetFrame(); + } +} + +#endif \ No newline at end of file diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp new file mode 100644 index 000000000..7f9b9e5d0 --- /dev/null +++ b/modules/tracking/src/tldDataset.cpp @@ -0,0 +1,101 @@ +#include "opencv2/tracking/tldDataset.hpp" + +namespace cv +{ + namespace tld + { + char tldRootPath[100]; + int frameNum = 0; + bool flagPNG = false; + + cv::Rect2d tld_InitDataset(int datasetInd, char* rootPath) + { + char* folderName = ""; + int x, y, w, h; + flagPNG = false; + + frameNum = 1; + + if (datasetInd == 1) { + folderName = "01_david"; + x = 165, y = 83; + w = 51; h = 54; + frameNum = 100; + } + if (datasetInd == 2) { + folderName = "02_jumping"; + x = 147, y = 110; + w = 33; h = 32; + } + if (datasetInd == 3) { + folderName = "03_pedestrian1"; + x = 47, y = 51; + w = 21; h = 36; + } + if (datasetInd == 4) { + folderName = "04_pedestrian2"; + x = 130, y = 134; + w = 21; h = 53; + } + if (datasetInd == 5) { + folderName = "05_pedestrian3"; + x = 154, y = 102; + w = 24; h = 52; + } + if (datasetInd == 6) { + folderName = "06_car"; + x = 142, y = 125; + w = 90; h = 39; + } + if (datasetInd == 7) { + folderName = "07_motocross"; + x = 290, y = 43; + w = 23; h = 40; + flagPNG = true; + } + if (datasetInd == 8) { + folderName = "08_volkswagen"; + x = 273, y = 77; + w = 27; h = 25; + } + if (datasetInd == 9) { + folderName = "09_carchase"; + x = 145, y = 84; + w = 54; h = 37; + } + if (datasetInd == 10){ + folderName = "10_panda"; + x = 58, y = 100; + w = 27; h = 22; + } + + strcpy(tldRootPath, rootPath); + strcat(tldRootPath, "\\"); + strcat(tldRootPath, folderName); + + + return cv::Rect2d(x, y, w, h); + } + + cv::Mat tld_getNextDatasetFrame() + { + char fullPath[100]; + char numStr[10]; + strcpy(fullPath, tldRootPath); + strcat(fullPath, "\\"); + if (frameNum < 10) strcat(fullPath, "0000"); + else if (frameNum < 100) strcat(fullPath, "000"); + else if (frameNum < 1000) strcat(fullPath, "00"); + else if (frameNum < 10000) strcat(fullPath, "0"); + + _itoa(frameNum, numStr, 10); + strcat(fullPath, numStr); + if (flagPNG) strcat(fullPath, ".png"); + else strcat(fullPath, ".jpg"); + frameNum++; + + return cv::imread(fullPath); + } + + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp new file mode 100644 index 000000000..2caf179ec --- /dev/null +++ b/modules/tracking/src/tldDetector.cpp @@ -0,0 +1,195 @@ +#include "tldDetector.hpp" + +namespace cv +{ + namespace tld + { + // Calculate offsets for classifiers + void TLDDetector::prepareClassifiers(int rowstep) + { + for (int i = 0; i < (int)classifiers.size(); i++) + classifiers[i].prepareClassifier(rowstep); + } + + // Calculate posterior probability, that the patch belongs to the current EC model + double TLDDetector::ensembleClassifierNum(const uchar* data) + { + double p = 0; + for (int k = 0; k < (int)classifiers.size(); k++) + p += classifiers[k].posteriorProbabilityFast(data); + p /= classifiers.size(); + return p; + } + + // Calculate Relative similarity of the patch (NN-Model) + double TLDDetector::Sr(const Mat_& patch) + { + double splus = 0.0, sminus = 0.0; + for (int i = 0; i < (int)(*positiveExamples).size(); i++) + splus = std::max(splus, 0.5 * (NCC((*positiveExamples)[i], patch) + 1.0)); + for (int i = 0; i < (int)(*negativeExamples).size(); i++) + sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); + if (splus + sminus == 0.0) + return 0.0; + return splus / (sminus + splus); + } + + // Calculate Conservative similarity of the patch (NN-Model) + double TLDDetector::Sc(const Mat_& patch) + { + double splus = 0.0, sminus = 0.0; + int med = getMedian((*timeStampsPositive)); + for (int i = 0; i < (int)(*positiveExamples).size(); i++) + { + if ((int)(*timeStampsPositive)[i] <= med) + splus = std::max(splus, 0.5 * (NCC((*positiveExamples)[i], patch) + 1.0)); + } + for (int i = 0; i < (int)(*negativeExamples).size(); i++) + sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); + if (splus + sminus == 0.0) + return 0.0; + return splus / (sminus + splus); + } + + // Generate Search Windows for detector from aspect ratio of initial BBs + void TLDDetector::generateScanGrid(int rows, int cols, Size initBox, std::vector& res, bool withScaling) + { + res.clear(); + //Scales step: SCALE_STEP; Translation steps: 10% of width & 10% of height; minSize: 20pix + for (double h = initBox.height, w = initBox.width; h < cols && w < rows;) + { + for (double x = 0; (x + w + 1.0) <= cols; x += (0.1 * w)) + { + for (double y = 0; (y + h + 1.0) <= rows; y += (0.1 * h)) + res.push_back(Rect2d(x, y, w, h)); + } + if (withScaling) + { + if (h <= initBox.height) + { + h /= SCALE_STEP; w /= SCALE_STEP; + if (h < 20 || w < 20) + { + h = initBox.height * SCALE_STEP; w = initBox.width * SCALE_STEP; + CV_Assert(h > initBox.height || w > initBox.width); + } + } + else + { + h *= SCALE_STEP; w *= SCALE_STEP; + } + } + else + { + break; + } + } + //dprintf(("%d rects in res\n", (int)res.size())); + } + + //Detection - returns most probable new target location (Max Sc) + bool TLDDetector::detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize) + { + patches.clear(); + + Mat resized_img, blurred_img; + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + img.copyTo(resized_img); + imgBlurred.copyTo(blurred_img); + int dx = initSize.width / 10, dy = initSize.height / 10; + Size2d size = img.size(); + double scale = 1.0; + int total = 0, pass = 0; + int npos = 0, nneg = 0; + double tmp = 0, maxSc = -5.0; + Rect2d maxScRect; + + //Detection part + //To fix: use precalculated BB + do + { + Mat_ intImgP, intImgP2; + computeIntegralImages(resized_img, intImgP, intImgP2); + + prepareClassifiers((int)blurred_img.step[0]); + for (int i = 0, imax = cvFloor((0.0 + resized_img.cols - initSize.width) / dx); i < imax; i++) + { + for (int j = 0, jmax = cvFloor((0.0 + resized_img.rows - initSize.height) / dy); j < jmax; j++) + { + LabeledPatch labPatch; + total++; + if (!patchVariance(intImgP, intImgP2, *originalVariance, Point(dx * i, dy * j), initSize)) + continue; + if (ensembleClassifierNum(&blurred_img.at(dy * j, dx * i)) <= ENSEMBLE_THRESHOLD) + continue; + pass++; + + labPatch.rect = Rect2d(dx * i * scale, dy * j * scale, initSize.width * scale, initSize.height * scale); + resample(resized_img, Rect2d(Point(dx * i, dy * j), initSize), standardPatch); + tmp = Sr(standardPatch); + + ////To fix: Check the paper, probably this cause wrong learning + // + labPatch.isObject = tmp > THETA_NN; + labPatch.shouldBeIntegrated = abs(tmp - THETA_NN) < 0.1; + patches.push_back(labPatch); + // + + if (!labPatch.isObject) + { + nneg++; + continue; + } + else + { + npos++; + } + tmp = Sc(standardPatch); + if (tmp > maxSc) + { + maxSc = tmp; + maxScRect = labPatch.rect; + } + } + } + + size.width /= SCALE_STEP; + size.height /= SCALE_STEP; + scale *= SCALE_STEP; + resize(img, resized_img, size, 0, 0, DOWNSCALE_MODE); + GaussianBlur(resized_img, blurred_img, GaussBlurKernelSize, 0.0f); + } while (size.width >= initSize.width && size.height >= initSize.height); + + if (maxSc < 0) + return false; + res = maxScRect; + return true; + } + + // Computes the variance of subimage given by box, with the help of two integral + // images intImgP and intImgP2 (sum of squares), which should be also provided. + bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size) + { + int x = (pt.x), y = (pt.y), width = (size.width), height = (size.height); + CV_Assert(0 <= x && (x + width) < intImgP.cols && (x + width) < intImgP2.cols); + CV_Assert(0 <= y && (y + height) < intImgP.rows && (y + height) < intImgP2.rows); + double p = 0, p2 = 0; + double A, B, C, D; + + A = intImgP(y, x); + B = intImgP(y, x + width); + C = intImgP(y + height, x); + D = intImgP(y + height, x + width); + p = (A + D - B - C) / (width * height); + + A = intImgP2(y, x); + B = intImgP2(y, x + width); + C = intImgP2(y + height, x); + D = intImgP2(y + height, x + width); + p2 = (A + D - B - C) / (width * height); + + return ((p2 - p * p) > VARIANCE_THRESHOLD * originalVariance); + } + + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp new file mode 100644 index 000000000..feead4826 --- /dev/null +++ b/modules/tracking/src/tldDetector.hpp @@ -0,0 +1,61 @@ +#ifndef OPENCV_TLD_DETECTOR +#define OPENCV_TLD_DETECTOR + +#include "precomp.hpp" +#include "tldEnsembleClassifier.hpp" +#include "tldUtils.hpp" + +namespace cv +{ + namespace tld + { + const int STANDARD_PATCH_SIZE = 15; + const int NEG_EXAMPLES_IN_INIT_MODEL = 300; + const int MAX_EXAMPLES_IN_MODEL = 500; + const int MEASURES_PER_CLASSIFIER = 13; + const int GRIDSIZE = 15; + const int DOWNSCALE_MODE = cv::INTER_LINEAR; + const double THETA_NN = 0.50; + const double CORE_THRESHOLD = 0.5; + const double SCALE_STEP = 1.2; + const double ENSEMBLE_THRESHOLD = 0.5; + const double VARIANCE_THRESHOLD = 0.5; + const double NEXPERT_THRESHOLD = 0.2; + + static const cv::Size GaussBlurKernelSize(3, 3); + + class TLDDetector + { + public: + TLDDetector(){} + ~TLDDetector(){} + + inline double ensembleClassifierNum(const uchar* data); + inline void prepareClassifiers(int rowstep); + double Sr(const Mat_& patch); + double Sc(const Mat_& patch); + + std::vector classifiers; + std::vector > *positiveExamples, *negativeExamples; + std::vector *timeStampsPositive, *timeStampsNegative; + double *originalVariance; + + static void generateScanGrid(int rows, int cols, Size initBox, std::vector& res, bool withScaling = false); + struct LabeledPatch + { + Rect2d rect; + bool isObject, shouldBeIntegrated; + }; + bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); + protected: + + + + friend class MyMouseCallbackDEBUG; + void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } + inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size); + }; + } +} + +#endif \ No newline at end of file diff --git a/modules/tracking/src/tldEnsembleClassifier.cpp b/modules/tracking/src/tldEnsembleClassifier.cpp new file mode 100644 index 000000000..3912d55ab --- /dev/null +++ b/modules/tracking/src/tldEnsembleClassifier.cpp @@ -0,0 +1,158 @@ +#include "tldEnsembleClassifier.hpp" + +namespace cv +{ + namespace tld + { + // Constructor + TLDEnsembleClassifier::TLDEnsembleClassifier(const std::vector& meas, int beg, int end) :lastStep_(-1) + { + int posSize = 1, mpc = end - beg; + for (int i = 0; i < mpc; i++) + posSize *= 2; + posAndNeg.assign(posSize, Point2i(0, 0)); + measurements.assign(meas.begin() + beg, meas.begin() + end); + offset.assign(mpc, Point2i(0, 0)); + } + + // Calculate measure locations from 15x15 grid on minSize patches + void TLDEnsembleClassifier::stepPrefSuff(std::vector& arr, int pos, int len, int gridSize) + { +#if 0 + int step = len / (gridSize - 1), pref = (len - step * (gridSize - 1)) / 2; + for (int i = 0; i < (int)(sizeof(x1) / sizeof(x1[0])); i++) + arr[i] = pref + arr[i] * step; +#else + int total = len - gridSize; + int quo = total / (gridSize - 1), rem = total % (gridSize - 1); + int smallStep = quo, bigStep = quo + 1; + int bigOnes = rem, smallOnes = gridSize - bigOnes - 1; + int bigOnes_front = bigOnes / 2, bigOnes_back = bigOnes - bigOnes_front; + for (int i = 0; i < (int)arr.size(); i++) + { + if (arr[i].val[pos] < bigOnes_back) + { + arr[i].val[pos] = (uchar)(arr[i].val[pos] * bigStep + arr[i].val[pos]); + continue; + } + if (arr[i].val[pos] < (bigOnes_front + smallOnes)) + { + arr[i].val[pos] = (uchar)(bigOnes_front * bigStep + (arr[i].val[pos] - bigOnes_front) * smallStep + arr[i].val[pos]); + continue; + } + if (arr[i].val[pos] < (bigOnes_front + smallOnes + bigOnes_back)) + { + arr[i].val[pos] = + (uchar)(bigOnes_front * bigStep + smallOnes * smallStep + + (arr[i].val[pos] - (bigOnes_front + smallOnes)) * bigStep + arr[i].val[pos]); + continue; + } + arr[i].val[pos] = (uchar)(len - 1); + } +#endif + } + + // Calculate offsets for classifier + void TLDEnsembleClassifier::prepareClassifier(int rowstep) + { + if (lastStep_ != rowstep) + { + lastStep_ = rowstep; + for (int i = 0; i < (int)offset.size(); i++) + { + offset[i].x = rowstep * measurements[i].val[2] + measurements[i].val[0]; + offset[i].y = rowstep * measurements[i].val[3] + measurements[i].val[1]; + } + } + } + + // Integrate patch into the Ensemble Classifier model + void TLDEnsembleClassifier::integrate(const Mat_& patch, bool isPositive) + { + int position = code(patch.data, (int)patch.step[0]); + if (isPositive) + posAndNeg[position].x++; + else + posAndNeg[position].y++; + } + + // Calculate posterior probability on the patch + double TLDEnsembleClassifier::posteriorProbability(const uchar* data, int rowstep) const + { + int position = code(data, rowstep); + double posNum = (double)posAndNeg[position].x, negNum = (double)posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + return 0.0; + else + return posNum / (posNum + negNum); + } + double TLDEnsembleClassifier::posteriorProbabilityFast(const uchar* data) const + { + int position = codeFast(data); + double posNum = (double)posAndNeg[position].x, negNum = (double)posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + return 0.0; + else + return posNum / (posNum + negNum); + } + + // Calculate the 13-bit fern index + int TLDEnsembleClassifier::codeFast(const uchar* data) const + { + int position = 0; + for (int i = 0; i < (int)measurements.size(); i++) + { + position = position << 1; + if (data[offset[i].x] < data[offset[i].y]) + position++; + } + return position; + } + int TLDEnsembleClassifier::code(const uchar* data, int rowstep) const + { + int position = 0; + for (int i = 0; i < (int)measurements.size(); i++) + { + position = position << 1; + if (*(data + rowstep * measurements[i].val[2] + measurements[i].val[0]) < + *(data + rowstep * measurements[i].val[3] + measurements[i].val[1])) + { + position++; + } + } + return position; + } + + // Create fern classifiers + int TLDEnsembleClassifier::makeClassifiers(Size size, int measurePerClassifier, int gridSize, + std::vector& classifiers) + { + + std::vector measurements; + + //Generate random measures for 10 ferns x 13 measures + for (int i = 0; i < 10*measurePerClassifier; i++) + { + Vec4b m; + m.val[0] = rand() % 15; + m.val[1] = rand() % 15; + m.val[2] = rand() % 15; + m.val[3] = rand() % 15; + measurements.push_back(m); + } + + //Warp measures to minSize patch coordinates + stepPrefSuff(measurements, 0, size.width, gridSize); + stepPrefSuff(measurements, 1, size.width, gridSize); + stepPrefSuff(measurements, 2, size.height, gridSize); + stepPrefSuff(measurements, 3, size.height, gridSize); + + //Compile fern classifiers + for (int i = 0, howMany = (int)measurements.size() / measurePerClassifier; i < howMany; i++) + classifiers.push_back(TLDEnsembleClassifier(measurements, i * measurePerClassifier, (i + 1) * measurePerClassifier)); + + return (int)classifiers.size(); + } + + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp new file mode 100644 index 000000000..446a0359c --- /dev/null +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -0,0 +1,29 @@ +#include +#include "precomp.hpp" + +namespace cv +{ + namespace tld + { + class TLDEnsembleClassifier + { + public: + static int makeClassifiers(Size size, int measurePerClassifier, int gridSize, std::vector& classifiers); + void integrate(const Mat_& patch, bool isPositive); + double posteriorProbability(const uchar* data, int rowstep) const; + double posteriorProbabilityFast(const uchar* data) const; + void prepareClassifier(int rowstep); + private: + TLDEnsembleClassifier(const std::vector& meas, int beg, int end); + static void stepPrefSuff(std::vector & arr, int pos, int len, int gridSize); + int code(const uchar* data, int rowstep) const; + int codeFast(const uchar* data) const; + std::vector posAndNeg; + std::vector measurements; + std::vector offset; + int lastStep_; + }; + + + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp new file mode 100644 index 000000000..212b42d41 --- /dev/null +++ b/modules/tracking/src/tldModel.cpp @@ -0,0 +1,236 @@ +#include "tldModel.hpp" + +namespace cv +{ + namespace tld + { + //Constructor + TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize) :minSize_(minSize), + timeStampPositiveNext(0), timeStampNegativeNext(0), params_(params), boundingBox_(boundingBox) + { + std::vector closest, scanGrid; + Mat scaledImg, blurredImg, image_blurred; + + //Create Detector + detector = Ptr(new TLDDetector()); + + //Propagate data to Detector + detector->positiveExamples = &positiveExamples; + detector->negativeExamples = &negativeExamples; + detector->timeStampsPositive = &timeStampsPositive; + detector->timeStampsNegative = &timeStampsNegative; + detector->originalVariance = &originalVariance_; + + //Calculate the variance in initial BB + originalVariance_ = variance(image(boundingBox)); + + //Find the scale + double scale = scaleAndBlur(image, cvRound(log(1.0 * boundingBox.width / (minSize.width)) / log(SCALE_STEP)), + scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); + GaussianBlur(image, image_blurred, GaussBlurKernelSize, 0.0); + TLDDetector::generateScanGrid(image.rows, image.cols, minSize_, scanGrid); + getClosestN(scanGrid, Rect2d(boundingBox.x / scale, boundingBox.y / scale, boundingBox.width / scale, + boundingBox.height / scale), 10, closest); + + Mat_ blurredPatch(minSize); + TLDEnsembleClassifier::makeClassifiers(minSize, MEASURES_PER_CLASSIFIER, GRIDSIZE, detector->classifiers); + + //Generate initial positive samples and put them to the model + positiveExamples.reserve(200); + for (int i = 0; i < (int)closest.size(); i++) + { + for (int j = 0; j < 20; j++) + { + Point2f center; + Size2f size; + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); + center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); + size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); + size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); + float angle = (float)rng.uniform(-10.0, 10.0); + + resample(scaledImg, RotatedRect(center, size, angle), standardPatch); + + for (int y = 0; y < standardPatch.rows; y++) + { + for (int x = 0; x < standardPatch.cols; x++) + { + standardPatch(x, y) += (uchar)rng.gaussian(5.0); + } + } + +#ifdef BLUR_AS_VADIM + GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); + resize(blurredPatch, blurredPatch, minSize); +#else + resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); +#endif + pushIntoModel(standardPatch, true); + for (int k = 0; k < (int)detector->classifiers.size(); k++) + detector->classifiers[k].integrate(blurredPatch, true); + } + } + + //Generate initial negative samples and put them to the model + TLDDetector::generateScanGrid(image.rows, image.cols, minSize, scanGrid, true); + negativeExamples.clear(); + negativeExamples.reserve(NEG_EXAMPLES_IN_INIT_MODEL); + std::vector indices; + indices.reserve(NEG_EXAMPLES_IN_INIT_MODEL); + while ((int)negativeExamples.size() < NEG_EXAMPLES_IN_INIT_MODEL) + { + int i = rng.uniform((int)0, (int)scanGrid.size()); + if (std::find(indices.begin(), indices.end(), i) == indices.end() && overlap(boundingBox, scanGrid[i]) < NEXPERT_THRESHOLD) + { + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + resample(image, scanGrid[i], standardPatch); + pushIntoModel(standardPatch, false); + + resample(image_blurred, scanGrid[i], blurredPatch); + for (int k = 0; k < (int)detector->classifiers.size(); k++) + detector->classifiers[k].integrate(blurredPatch, false); + } + } + //dprintf(("positive patches: %d\nnegative patches: %d\n", (int)positiveExamples.size(), (int)negativeExamples.size())); + } + + + void TrackerTLDModel::integrateRelabeled(Mat& img, Mat& imgBlurred, const std::vector& patches) + { + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(minSize_); + int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; + for (int k = 0; k < (int)patches.size(); k++) + { + if (patches[k].shouldBeIntegrated) + { + resample(img, patches[k].rect, standardPatch); + if (patches[k].isObject) + { + positiveIntoModel++; + pushIntoModel(standardPatch, true); + } + else + { + negativeIntoModel++; + pushIntoModel(standardPatch, false); + } + } + +#ifdef CLOSED_LOOP + if (patches[k].shouldBeIntegrated || !patches[k].isPositive) +#else + if (patches[k].shouldBeIntegrated) +#endif + { + resample(imgBlurred, patches[k].rect, blurredPatch); + if (patches[k].isObject) + positiveIntoEnsemble++; + else + negativeIntoEnsemble++; + for (int i = 0; i < (int)detector->classifiers.size(); i++) + detector->classifiers[i].integrate(blurredPatch, patches[k].isObject); + } + } + /* + if( negativeIntoModel > 0 ) + dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); + if( positiveIntoModel > 0) + dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); + if( negativeIntoEnsemble > 0 ) + dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); + if( positiveIntoEnsemble > 0 ) + dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); + dfprintf((stdout, "\n"));*/ + + } + + void TrackerTLDModel::integrateAdditional(const std::vector >& eForModel, const std::vector >& eForEnsemble, bool isPositive) + { + int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; + for (int k = 0; k < (int)eForModel.size(); k++) + { + double sr = detector->Sr(eForModel[k]); + if ((sr > THETA_NN) != isPositive) + { + if (isPositive) + { + positiveIntoModel++; + pushIntoModel(eForModel[k], true); + } + else + { + negativeIntoModel++; + pushIntoModel(eForModel[k], false); + } + } + double p = 0; + for (int i = 0; i < (int)detector->classifiers.size(); i++) + p += detector->classifiers[i].posteriorProbability(eForEnsemble[k].data, (int)eForEnsemble[k].step[0]); + p /= detector->classifiers.size(); + if ((p > ENSEMBLE_THRESHOLD) != isPositive) + { + if (isPositive) + positiveIntoEnsemble++; + else + negativeIntoEnsemble++; + for (int i = 0; i < (int)detector->classifiers.size(); i++) + detector->classifiers[i].integrate(eForEnsemble[k], isPositive); + } + } + /* + if( negativeIntoModel > 0 ) + dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); + if( positiveIntoModel > 0 ) + dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); + if( negativeIntoEnsemble > 0 ) + dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); + if( positiveIntoEnsemble > 0 ) + dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); + dfprintf((stdout, "\n"));*/ + } + + //Push the patch to the model + void TrackerTLDModel::pushIntoModel(const Mat_& example, bool positive) + { + std::vector >* proxyV; + int* proxyN; + std::vector* proxyT; + if (positive) + { + proxyV = &positiveExamples; + proxyN = &timeStampPositiveNext; + proxyT = &timeStampsPositive; + } + else + { + proxyV = &negativeExamples; + proxyN = &timeStampNegativeNext; + proxyT = &timeStampsNegative; + } + if ((int)proxyV->size() < MAX_EXAMPLES_IN_MODEL) + { + proxyV->push_back(example); + proxyT->push_back(*proxyN); + } + else + { + int index = rng.uniform((int)0, (int)proxyV->size()); + (*proxyV)[index] = example; + (*proxyT)[index] = (*proxyN); + } + (*proxyN)++; + } + + void TrackerTLDModel::printme(FILE* port) + { + //dfprintf((port, "TrackerTLDModel:\n")); + //dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); + //dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); + } + + + + + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldModel.hpp b/modules/tracking/src/tldModel.hpp new file mode 100644 index 000000000..a0ac0a349 --- /dev/null +++ b/modules/tracking/src/tldModel.hpp @@ -0,0 +1,50 @@ +#ifndef OPENCV_TLD_MODEL +#define OPENCV_TLD_MODEL + +#include "precomp.hpp" +#include "tldDetector.hpp" +#include "tldUtils.hpp" + +namespace cv +{ + namespace tld + { + + + + + class TrackerTLDModel : public TrackerModel + { + public: + TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize); + Rect2d getBoundingBox(){ return boundingBox_; } + void setBoudingBox(Rect2d boundingBox){ boundingBox_ = boundingBox; } + void integrateRelabeled(Mat& img, Mat& imgBlurred, const std::vector& patches); + void integrateAdditional(const std::vector >& eForModel, const std::vector >& eForEnsemble, bool isPositive); + Size getMinSize(){ return minSize_; } + void printme(FILE* port = stdout); + Ptr detector; + + std::vector > positiveExamples, negativeExamples; + std::vector timeStampsPositive, timeStampsNegative; + int timeStampPositiveNext, timeStampNegativeNext; + double originalVariance_; + + double getOriginalVariance(){ return originalVariance_; } + + protected: + Size minSize_; + + TrackerTLD::Params params_; + void pushIntoModel(const Mat_& example, bool positive); + void modelEstimationImpl(const std::vector& /*responses*/){} + void modelUpdateImpl(){} + Rect2d boundingBox_; + RNG rng; + + }; + + } +} + +#endif \ No newline at end of file diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp new file mode 100644 index 000000000..c55f281b4 --- /dev/null +++ b/modules/tracking/src/tldTracker.cpp @@ -0,0 +1,299 @@ +/*/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "tldTracker.hpp" + + +namespace cv +{ + +Ptr TrackerTLD::createTracker(const TrackerTLD::Params ¶meters) +{ + return Ptr(new tld::TrackerTLDImpl(parameters)); +} + +namespace tld +{ + +TrackerTLDImpl::TrackerTLDImpl(const TrackerTLD::Params ¶meters) : + params( parameters ) +{ + isInit = false; + trackerProxy = Ptr > + (new TrackerProxyImpl()); +} + +void TrackerTLDImpl::read(const cv::FileNode& fn) +{ + params.read( fn ); +} + +void TrackerTLDImpl::write(cv::FileStorage& fs) const +{ + params.write( fs ); +} + +bool TrackerTLDImpl::initImpl(const Mat& image, const Rect2d& boundingBox) +{ + Mat image_gray; + trackerProxy->init(image, boundingBox); + cvtColor( image, image_gray, COLOR_BGR2GRAY ); + data = Ptr(new Data(boundingBox)); + double scale = data->getScale(); + Rect2d myBoundingBox = boundingBox; + if( scale > 1.0 ) + { + Mat image_proxy; + resize(image_gray, image_proxy, Size(cvRound(image.cols * scale), cvRound(image.rows * scale)), 0, 0, DOWNSCALE_MODE); + image_proxy.copyTo(image_gray); + myBoundingBox.x *= scale; + myBoundingBox.y *= scale; + myBoundingBox.width *= scale; + myBoundingBox.height *= scale; + } + model = Ptr(new TrackerTLDModel(params, image_gray, myBoundingBox, data->getMinSize())); + TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(model)); + + + data->confident = false; + data->failedLastTime = false; + + return true; +} + +bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) +{ + Mat image_gray, image_blurred, imageForDetector; + cvtColor( image, image_gray, COLOR_BGR2GRAY ); + double scale = data->getScale(); + if( scale > 1.0 ) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, GaussBlurKernelSize, 0.0); + TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(model)); + data->frameNum++; + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + std::vector detectorResults; + //best overlap around 92% + + std::vector candidates; + std::vector candidatesRes; + bool trackerNeedsReInit = false; + for( int i = 0; i < 2; i++ ) + { + Rect2d tmpCandid = boundingBox; + if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || + ((i == 1) && (tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize())))) + { + candidates.push_back(tmpCandid); + if( i == 0 ) + resample(image_gray, tmpCandid, standardPatch); + else + resample(imageForDetector, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + } + else + { + if( i == 0 ) + trackerNeedsReInit = true; + } + } + + std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + + //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); + //for( int i = 0; i < (int)candidatesRes.size(); i++ ) + //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); + //data->printme(); + //tldModel->printme(stdout); + + if( it == candidatesRes.end() ) + { + data->confident = false; + data->failedLastTime = true; + return false; + } + else + { + boundingBox = candidates[it - candidatesRes.begin()]; + data->failedLastTime = false; + if( trackerNeedsReInit || it != candidatesRes.begin() ) + trackerProxy->init(image, boundingBox); + } + +#if 1 + if( it != candidatesRes.end() ) + { + resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); + //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) + //dfprintf((stderr, "detector WON\n")); + } + else + { + //dfprintf((stderr, "%d x x\n", data->frameNum)); + } +#endif + + if( *it > CORE_THRESHOLD ) + data->confident = true; + + if( data->confident ) + { + Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, params, data->getMinSize()); + Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, params); + std::vector > examplesForModel, examplesForEnsemble; + examplesForModel.reserve(100); examplesForEnsemble.reserve(100); + int negRelabeled = 0; + for( int i = 0; i < (int)detectorResults.size(); i++ ) + { + bool expertResult; + if( detectorResults[i].isObject ) + { + expertResult = nExpert(detectorResults[i].rect); + if( expertResult != detectorResults[i].isObject ) + negRelabeled++; + } + else + { + expertResult = pExpert(detectorResults[i].rect); + } + + detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); + detectorResults[i].isObject = expertResult; + } + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + //dprintf(("%d relabeled by nExpert\n", negRelabeled)); + pExpert.additionalExamples(examplesForModel, examplesForEnsemble); + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); + examplesForModel.clear(); examplesForEnsemble.clear(); + nExpert.additionalExamples(examplesForModel, examplesForEnsemble); + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); + } + else + { +#ifdef CLOSED_LOOP + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); +#endif + } + + return true; +} + + +int TrackerTLDImpl::Pexpert::additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble) +{ + examplesForModel.clear(); examplesForEnsemble.clear(); + examplesForModel.reserve(100); examplesForEnsemble.reserve(100); + + std::vector closest, scanGrid; + Mat scaledImg, blurredImg; + + double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)), + scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); + TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid); + getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest); + + for( int i = 0; i < (int)closest.size(); i++ ) + { + for( int j = 0; j < 10; j++ ) + { + Point2f center; + Size2f size; + Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(initSize_); + center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); + center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); + size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); + size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); + float angle = (float)rng.uniform(-5.0, 5.0); + + for( int y = 0; y < standardPatch.rows; y++ ) + { + for( int x = 0; x < standardPatch.cols; x++ ) + { + standardPatch(x, y) += (uchar)rng.gaussian(5.0); + } + } +#ifdef BLUR_AS_VADIM + GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); + resize(blurredPatch, blurredPatch, initSize_); +#else + resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); +#endif + resample(scaledImg, RotatedRect(center, size, angle), standardPatch); + examplesForModel.push_back(standardPatch); + examplesForEnsemble.push_back(blurredPatch); + } + } + return 0; +} + +bool TrackerTLDImpl::Nexpert::operator()(Rect2d box) +{ + if( overlap(resultBox_, box) < NEXPERT_THRESHOLD ) + return false; + else + return true; +} + +Data::Data(Rect2d initBox) +{ + double minDim = std::min(initBox.width, initBox.height); + scale = 20.0 / minDim; + minSize.width = (int)(initBox.width * 20.0 / minDim); + minSize.height = (int)(initBox.height * 20.0 / minDim); + frameNum = 0; + //dprintf(("minSize = %dx%d\n", minSize.width, minSize.height)); +} + +void Data::printme(FILE* port) +{ + dfprintf((port, "Data:\n")); + dfprintf((port, "\tframeNum = %d\n", frameNum)); + dfprintf((port, "\tconfident = %s\n", confident?"true":"false")); + dfprintf((port, "\tfailedLastTime = %s\n", failedLastTime?"true":"false")); + dfprintf((port, "\tminSize = %dx%d\n", minSize.width, minSize.height)); +} + +} + +} diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp new file mode 100644 index 000000000..1cc9e4863 --- /dev/null +++ b/modules/tracking/src/tldTracker.hpp @@ -0,0 +1,177 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ +#ifndef OPENCV_TLD_TRACKER +#define OPENCV_TLD_TRACKER + +#include "precomp.hpp" +#include "opencv2/video/tracking.hpp" +#include "opencv2/imgproc.hpp" +#include "tldModel.hpp" +#include +#include + +namespace cv +{ + +TrackerTLD::Params::Params(){} + +void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} + +void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} + +namespace tld +{ + + +class TrackerProxy +{ +public: + virtual bool init(const Mat& image, const Rect2d& boundingBox) = 0; + virtual bool update(const Mat& image, Rect2d& boundingBox) = 0; + virtual ~TrackerProxy(){} +}; + + +class MyMouseCallbackDEBUG +{ +public: + MyMouseCallbackDEBUG(Mat& img, Mat& imgBlurred, TLDDetector* detector) :img_(img), imgBlurred_(imgBlurred), detector_(detector){} + static void onMouse(int event, int x, int y, int, void* obj){ ((MyMouseCallbackDEBUG*)obj)->onMouse(event, x, y); } + MyMouseCallbackDEBUG& operator = (const MyMouseCallbackDEBUG& /*other*/){ return *this; } +private: + void onMouse(int event, int x, int y); + Mat& img_, imgBlurred_; + TLDDetector* detector_; +}; + + +class Data +{ +public: + Data(Rect2d initBox); + Size getMinSize(){ return minSize; } + double getScale(){ return scale; } + bool confident; + bool failedLastTime; + int frameNum; + void printme(FILE* port = stdout); +private: + double scale; + Size minSize; +}; + +template +class TrackerProxyImpl : public TrackerProxy +{ +public: + TrackerProxyImpl(Tparams params = Tparams()) :params_(params){} + bool init(const Mat& image, const Rect2d& boundingBox) + { + trackerPtr = T::createTracker(); + return trackerPtr->init(image, boundingBox); + } + bool update(const Mat& image, Rect2d& boundingBox) + { + return trackerPtr->update(image, boundingBox); + } +private: + Ptr trackerPtr; + Tparams params_; + Rect2d boundingBox_; +}; + + +#define BLUR_AS_VADIM +#undef CLOSED_LOOP + +class TrackerTLDImpl : public TrackerTLD +{ +public: + TrackerTLDImpl(const TrackerTLD::Params ¶meters = TrackerTLD::Params()); + void read(const FileNode& fn); + void write(FileStorage& fs) const; + +protected: + class Pexpert + { + public: + Pexpert(const Mat& img_in, const Mat& imgBlurred_in, Rect2d& resultBox_in, + const TLDDetector* detector_in, TrackerTLD::Params params_in, Size initSize_in) : + img_(img_in), imgBlurred_(imgBlurred_in), resultBox_(resultBox_in), detector_(detector_in), params_(params_in), initSize_(initSize_in){} + bool operator()(Rect2d /*box*/){ return false; } + int additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble); + protected: + Pexpert(){} + Mat img_, imgBlurred_; + Rect2d resultBox_; + const TLDDetector* detector_; + TrackerTLD::Params params_; + RNG rng; + Size initSize_; + }; + + class Nexpert : public Pexpert + { + public: + Nexpert(const Mat& img_in, Rect2d& resultBox_in, const TLDDetector* detector_in, TrackerTLD::Params params_in) + { + img_ = img_in; resultBox_ = resultBox_in; detector_ = detector_in; params_ = params_in; + } + bool operator()(Rect2d box); + int additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble) + { + examplesForModel.clear(); examplesForEnsemble.clear(); return 0; + } + }; + + bool initImpl(const Mat& image, const Rect2d& boundingBox); + bool updateImpl(const Mat& image, Rect2d& boundingBox); + + TrackerTLD::Params params; + Ptr data; + Ptr trackerProxy; + +}; + +} +} + +#endif \ No newline at end of file diff --git a/modules/tracking/src/tld_utils.cpp b/modules/tracking/src/tldUtils.cpp similarity index 61% rename from modules/tracking/src/tld_utils.cpp rename to modules/tracking/src/tldUtils.cpp index e534072ba..291e5f2d5 100644 --- a/modules/tracking/src/tld_utils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -39,20 +39,15 @@ // //M*/ -#include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc.hpp" -#include "time.h" -#include -#include -#include -#include -#include "tld_tracker.hpp" +#include "tldUtils.hpp" -namespace cv {namespace tld + +namespace cv +{ +namespace tld { -//debug functions and variables +//Debug functions and variables Rect2d etalon(14.0, 110.0, 20.0, 20.0); void drawWithRects(const Mat& img, std::vector& blackOnes, Rect2d whiteOne) { @@ -95,7 +90,6 @@ void myassert(const Mat& img) } dprintf(("black: %d out of %d (%f)\n", count, img.rows * img.cols, 1.0 * count / img.rows / img.cols)); } - void printPatch(const Mat_& standardPatch) { for( int i = 0; i < standardPatch.rows; i++ ) @@ -105,7 +99,6 @@ void printPatch(const Mat_& standardPatch) dprintf(("\n")); } } - std::string type2str(const Mat& mat) { int type = mat.type(); @@ -131,7 +124,7 @@ std::string type2str(const Mat& mat) return r; } -//generic functions +//Scale & Blur image using scale Indx double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep) { double dScale = 1.0; @@ -142,6 +135,8 @@ double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blur GaussianBlur(scaledImg, blurredImg, GaussBlurKernelSize, 0.0); return dScale; } + +//Find N-closest BB to the target void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res) { if( n >= (int)scanGrid.size() ) @@ -180,6 +175,7 @@ void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector< } } +//Calculate patch variance double variance(const Mat& img) { double p = 0, p2 = 0; @@ -196,6 +192,7 @@ double variance(const Mat& img) return p2 - p * p; } +//Normalized Correlation Coefficient double NCC(const Mat_& patch1, const Mat_& patch2) { CV_Assert( patch1.rows == patch2.rows ); @@ -217,6 +214,7 @@ double NCC(const Mat_& patch1, const Mat_& patch2) double ares = (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; return ares; } + int getMedian(const std::vector& values, int size) { if( size == -1 ) @@ -229,6 +227,7 @@ int getMedian(const std::vector& values, int size) return copy[(size - 1) / 2]; } +//Overlap between two BB double overlap(const Rect2d& r1, const Rect2d& r2) { double a1 = r1.area(), a2 = r2.area(), a0 = (r1&r2).area(); @@ -251,6 +250,7 @@ void resample(const Mat& img, const RotatedRect& r2, Mat_& samples) b.copyTo(M.colRange(Range(2, 3))); warpAffine(img, samples, M, samples.size()); } + void resample(const Mat& img, const Rect2d& r2, Mat_& samples) { Mat_ M(2, 3); @@ -259,146 +259,5 @@ void resample(const Mat& img, const Rect2d& r2, Mat_& samples) warpAffine(img, samples, M, samples.size()); } -//other stuff -void TLDEnsembleClassifier::stepPrefSuff(std::vector& arr, int pos, int len, int gridSize) -{ -#if 0 - int step = len / (gridSize - 1), pref = (len - step * (gridSize - 1)) / 2; - for( int i = 0; i < (int)(sizeof(x1) / sizeof(x1[0])); i++ ) - arr[i] = pref + arr[i] * step; -#else - int total = len - gridSize; - int quo = total / (gridSize - 1), rem = total % (gridSize - 1); - int smallStep = quo, bigStep = quo + 1; - int bigOnes = rem, smallOnes = gridSize - bigOnes - 1; - int bigOnes_front = bigOnes / 2, bigOnes_back = bigOnes - bigOnes_front; - for( int i = 0; i < (int)arr.size(); i++ ) - { - if( arr[i].val[pos] < bigOnes_back ) - { - arr[i].val[pos] = (uchar)(arr[i].val[pos] * bigStep + arr[i].val[pos]); - continue; - } - if( arr[i].val[pos] < (bigOnes_front + smallOnes) ) - { - arr[i].val[pos] = (uchar)(bigOnes_front * bigStep + (arr[i].val[pos] - bigOnes_front) * smallStep + arr[i].val[pos]); - continue; - } - if( arr[i].val[pos] < (bigOnes_front + smallOnes + bigOnes_back) ) - { - arr[i].val[pos] = - (uchar)(bigOnes_front * bigStep + smallOnes * smallStep + - (arr[i].val[pos] - (bigOnes_front + smallOnes)) * bigStep + arr[i].val[pos]); - continue; - } - arr[i].val[pos] = (uchar)(len - 1); - } -#endif -} -void TLDEnsembleClassifier::prepareClassifier(int rowstep) -{ - if( lastStep_ != rowstep ) - { - lastStep_ = rowstep; - for( int i = 0; i < (int)offset.size(); i++ ) - { - offset[i].x = rowstep * measurements[i].val[2] + measurements[i].val[0]; - offset[i].y = rowstep * measurements[i].val[3] + measurements[i].val[1]; - } - } -} -TLDEnsembleClassifier::TLDEnsembleClassifier(const std::vector& meas, int beg, int end):lastStep_(-1) -{ - int posSize = 1, mpc = end - beg; - for( int i = 0; i < mpc; i++ ) - posSize *= 2; - posAndNeg.assign(posSize, Point2i(0, 0)); - measurements.assign(meas.begin() + beg, meas.begin() + end); - offset.assign(mpc, Point2i(0, 0)); -} -void TLDEnsembleClassifier::integrate(const Mat_& patch, bool isPositive) -{ - int position = code(patch.data, (int)patch.step[0]); - if( isPositive ) - posAndNeg[position].x++; - else - posAndNeg[position].y++; -} -double TLDEnsembleClassifier::posteriorProbability(const uchar* data, int rowstep) const -{ - int position = code(data, rowstep); - double posNum = (double)posAndNeg[position].x, negNum = (double)posAndNeg[position].y; - if( posNum == 0.0 && negNum == 0.0 ) - return 0.0; - else - return posNum / (posNum + negNum); -} -double TLDEnsembleClassifier::posteriorProbabilityFast(const uchar* data) const -{ - int position = codeFast(data); - double posNum = (double)posAndNeg[position].x, negNum = (double)posAndNeg[position].y; - if( posNum == 0.0 && negNum == 0.0 ) - return 0.0; - else - return posNum / (posNum + negNum); -} -int TLDEnsembleClassifier::codeFast(const uchar* data) const -{ - int position = 0; - for( int i = 0; i < (int)measurements.size(); i++ ) - { - position = position << 1; - if( data[offset[i].x] < data[offset[i].y] ) - position++; - } - return position; -} -int TLDEnsembleClassifier::code(const uchar* data, int rowstep) const -{ - int position = 0; - for( int i = 0; i < (int)measurements.size(); i++ ) - { - position = position << 1; - if( *(data + rowstep * measurements[i].val[2] + measurements[i].val[0]) < - *(data + rowstep * measurements[i].val[3] + measurements[i].val[1]) ) - { - position++; - } - } - return position; -} -int TLDEnsembleClassifier::makeClassifiers(Size size, int measurePerClassifier, int gridSize, - std::vector& classifiers) -{ - - std::vector measurements; - - for( int i = 0; i < gridSize; i++ ) - { - for( int j = 0; j < gridSize; j++ ) - { - for( int k = 0; k < j; k++ ) - { - Vec4b m; - m.val[0] = m.val[2] = (uchar)i; - m.val[1] = (uchar)j; m.val[3] = (uchar)k; - measurements.push_back(m); - m.val[1] = m.val[3] = (uchar)i; - m.val[0] = (uchar)j; m.val[2] = (uchar)k; - measurements.push_back(m); - } - } - } - random_shuffle(measurements.begin(), measurements.end()); - - stepPrefSuff(measurements, 0, size.width, gridSize); - stepPrefSuff(measurements, 1, size.width, gridSize); - stepPrefSuff(measurements, 2, size.height, gridSize); - stepPrefSuff(measurements, 3, size.height, gridSize); - - for( int i = 0, howMany = (int)measurements.size() / measurePerClassifier; i < howMany; i++ ) - classifiers.push_back(TLDEnsembleClassifier(measurements, i * measurePerClassifier, (i + 1) * measurePerClassifier)); - return (int)classifiers.size(); -} }} diff --git a/modules/tracking/src/tldUtils.hpp b/modules/tracking/src/tldUtils.hpp new file mode 100644 index 000000000..9473ac9f6 --- /dev/null +++ b/modules/tracking/src/tldUtils.hpp @@ -0,0 +1,62 @@ +#ifndef OPENCV_TLD_UTILS +#define OPENCV_TLD_UTILS + +#include "precomp.hpp" +#include "opencv2/highgui.hpp" + +namespace cv +{ + namespace tld + { + //debug functions and variables + #define ALEX_DEBUG + #ifdef ALEX_DEBUG + #define dfprintf(x) fprintf x + #define dprintf(x) printf x + #else + #define dfprintf(x) + #define dprintf(x) + #endif + #define MEASURE_TIME(a)\ + {\ + clock_t start; float milisec = 0.0; \ + start = clock(); {a} milisec = 1000.0 * (clock() - start) / CLOCKS_PER_SEC; \ + dprintf(("%-90s took %f milis\n", #a, milisec));\ + } + #define HERE dprintf(("line %d\n", __LINE__)); fflush(stderr); + #define START_TICK(name)\ + { \ + clock_t start; double milisec = 0.0; start = clock(); + #define END_TICK(name) milisec = 1000.0 * (clock() - start) / CLOCKS_PER_SEC; \ + dprintf(("%s took %f milis\n", name, milisec)); \ + } + extern Rect2d etalon; + + void myassert(const Mat& img); + void printPatch(const Mat_& standardPatch); + std::string type2str(const Mat& mat); + void drawWithRects(const Mat& img, std::vector& blackOnes, Rect2d whiteOne = Rect2d(-1.0, -1.0, -1.0, -1.0)); + void drawWithRects(const Mat& img, std::vector& blackOnes, std::vector& whiteOnes, String fileName = ""); + + //aux functions and variables + template inline T CLIP(T x, T a, T b){ return std::min(std::max(x, a), b); } + /** Computes overlap between the two given rectangles. Overlap is computed as ratio of rectangles' intersection to that + * of their union.*/ + double overlap(const Rect2d& r1, const Rect2d& r2); + /** Resamples the area surrounded by r2 in img so it matches the size of samples, where it is written.*/ + void resample(const Mat& img, const RotatedRect& r2, Mat_& samples); + /** Specialization of resample() for rectangles without retation for better performance and simplicity.*/ + void resample(const Mat& img, const Rect2d& r2, Mat_& samples); + /** Computes the variance of single given image.*/ + double variance(const Mat& img); + /** Computes normalized corellation coefficient between the two patches (they should be + * of the same size).*/ + double NCC(const Mat_& patch1, const Mat_& patch2); + void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res); + double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep); + int getMedian(const std::vector& values, int size = -1); + + } +} + +#endif \ No newline at end of file diff --git a/modules/tracking/src/tld_tracker.cpp b/modules/tracking/src/tld_tracker.cpp deleted file mode 100644 index 695bf1e65..000000000 --- a/modules/tracking/src/tld_tracker.cpp +++ /dev/null @@ -1,951 +0,0 @@ -/*/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc.hpp" -#include "time.h" -#include -#include -#include "tld_tracker.hpp" -#include "opencv2/highgui.hpp" - -/* - * FIXME(optimize): - * no median - * direct formula in resamples - * FIXME(issues) - * THETA_NN 0.5<->0.6 dramatic change vs video 6 !! - * TODO(features) - * benchmark: two streams of photos -->better video - * (try inter_area for resize) - * TODO: - * fix pushbot->pick commits->compare_branches->all in 1->resubmit - * || video(0.5<->0.6) -->debug if box size is less than 20 - * perfect PN - * -* vadim: -* ?3. comment each function/method -* 5. empty lines to separate logical... -* 6. comment logical sections -* 11. group decls logically, order of statements -* -* ?10. all in one class -* todo: -* initializer lists; -*/ - -/* design decisions: - */ - -namespace cv -{ - -namespace tld -{ - -const int STANDARD_PATCH_SIZE = 15; -const int NEG_EXAMPLES_IN_INIT_MODEL = 300; -const int MAX_EXAMPLES_IN_MODEL = 500; -const int MEASURES_PER_CLASSIFIER = 13; -const int GRIDSIZE = 15; -const int DOWNSCALE_MODE = cv::INTER_LINEAR; -const double THETA_NN = 0.50; -const double CORE_THRESHOLD = 0.5; -const double SCALE_STEP = 1.2; -const double ENSEMBLE_THRESHOLD = 0.5; -const double VARIANCE_THRESHOLD = 0.5; -const double NEXPERT_THRESHOLD = 0.2; -#define BLUR_AS_VADIM -#undef CLOSED_LOOP -static const cv::Size GaussBlurKernelSize(3, 3); - -class TLDDetector; -class MyMouseCallbackDEBUG -{ -public: - MyMouseCallbackDEBUG(Mat& img, Mat& imgBlurred, TLDDetector* detector):img_(img), imgBlurred_(imgBlurred), detector_(detector){} - static void onMouse(int event, int x, int y, int, void* obj){ ((MyMouseCallbackDEBUG*)obj)->onMouse(event, x, y); } - MyMouseCallbackDEBUG& operator = (const MyMouseCallbackDEBUG& /*other*/){ return *this; } -private: - void onMouse(int event, int x, int y); - Mat& img_, imgBlurred_; - TLDDetector* detector_; -}; - -class Data -{ -public: - Data(Rect2d initBox); - Size getMinSize(){ return minSize; } - double getScale(){ return scale; } - bool confident; - bool failedLastTime; - int frameNum; - void printme(FILE* port = stdout); -private: - double scale; - Size minSize; -}; - -class TLDDetector -{ -public: - TLDDetector(const TrackerTLD::Params& params, Ptr model_in):model(model_in), params_(params){} - ~TLDDetector(){} - static void generateScanGrid(int rows, int cols, Size initBox, std::vector& res, bool withScaling = false); - struct LabeledPatch - { - Rect2d rect; - bool isObject, shouldBeIntegrated; - }; - bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches); -protected: - friend class MyMouseCallbackDEBUG; - Ptr model; - void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } - inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size); - TrackerTLD::Params params_; -}; - -template -class TrackerProxyImpl : public TrackerProxy -{ -public: - TrackerProxyImpl(Tparams params = Tparams()):params_(params){} - bool init(const Mat& image, const Rect2d& boundingBox) - { - trackerPtr = T::createTracker(); - return trackerPtr->init(image, boundingBox); - } - bool update(const Mat& image, Rect2d& boundingBox) - { - return trackerPtr->update(image, boundingBox); - } -private: - Ptr trackerPtr; - Tparams params_; - Rect2d boundingBox_; -}; - -class TrackerTLDModel : public TrackerModel -{ -public: - TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize); - Rect2d getBoundingBox(){ return boundingBox_; } - void setBoudingBox(Rect2d boundingBox){ boundingBox_ = boundingBox; } - double getOriginalVariance(){ return originalVariance_; } - inline double ensembleClassifierNum(const uchar* data); - inline void prepareClassifiers(int rowstep); - double Sr(const Mat_& patch); - double Sc(const Mat_& patch); - void integrateRelabeled(Mat& img, Mat& imgBlurred, const std::vector& patches); - void integrateAdditional(const std::vector >& eForModel, const std::vector >& eForEnsemble, bool isPositive); - Size getMinSize(){ return minSize_; } - void printme(FILE* port = stdout); - -protected: - Size minSize_; - int timeStampPositiveNext, timeStampNegativeNext; - TrackerTLD::Params params_; - void pushIntoModel(const Mat_& example, bool positive); - void modelEstimationImpl( const std::vector& /*responses*/ ){} - void modelUpdateImpl(){} - Rect2d boundingBox_; - double originalVariance_; - std::vector > positiveExamples, negativeExamples; - std::vector timeStampsPositive, timeStampsNegative; - RNG rng; - std::vector classifiers; -}; - -class TrackerTLDImpl : public TrackerTLD -{ -public: - TrackerTLDImpl(const TrackerTLD::Params ¶meters = TrackerTLD::Params()); - void read(const FileNode& fn); - void write(FileStorage& fs) const; - -protected: - class Pexpert - { - public: - Pexpert(const Mat& img_in, const Mat& imgBlurred_in, Rect2d& resultBox_in, - const TLDDetector* detector_in, TrackerTLD::Params params_in, Size initSize_in): - img_(img_in), imgBlurred_(imgBlurred_in), resultBox_(resultBox_in), detector_(detector_in), params_(params_in), initSize_(initSize_in){} - bool operator()(Rect2d /*box*/){ return false; } - int additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble); - protected: - Pexpert(){} - Mat img_, imgBlurred_; - Rect2d resultBox_; - const TLDDetector* detector_; - TrackerTLD::Params params_; - RNG rng; - Size initSize_; - }; - - class Nexpert : public Pexpert - { - public: - Nexpert(const Mat& img_in, Rect2d& resultBox_in, const TLDDetector* detector_in, TrackerTLD::Params params_in) - { - img_ = img_in; resultBox_ = resultBox_in; detector_ = detector_in; params_ = params_in; - } - bool operator()(Rect2d box); - int additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble) - { - examplesForModel.clear(); examplesForEnsemble.clear(); return 0; - } - }; - - bool initImpl(const Mat& image, const Rect2d& boundingBox); - bool updateImpl(const Mat& image, Rect2d& boundingBox); - - TrackerTLD::Params params; - Ptr data; - Ptr trackerProxy; - Ptr detector; -}; - -} - -TrackerTLD::Params::Params(){} - -void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} - -void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} - -Ptr TrackerTLD::createTracker(const TrackerTLD::Params ¶meters) -{ - return Ptr(new tld::TrackerTLDImpl(parameters)); -} - -namespace tld -{ - -TrackerTLDImpl::TrackerTLDImpl(const TrackerTLD::Params ¶meters) : - params( parameters ) -{ - isInit = false; - trackerProxy = Ptr > - (new TrackerProxyImpl()); -} - -void TrackerTLDImpl::read(const cv::FileNode& fn) -{ - params.read( fn ); -} - -void TrackerTLDImpl::write(cv::FileStorage& fs) const -{ - params.write( fs ); -} - -bool TrackerTLDImpl::initImpl(const Mat& image, const Rect2d& boundingBox) -{ - Mat image_gray; - trackerProxy->init(image, boundingBox); - cvtColor( image, image_gray, COLOR_BGR2GRAY ); - data = Ptr(new Data(boundingBox)); - double scale = data->getScale(); - Rect2d myBoundingBox = boundingBox; - if( scale > 1.0 ) - { - Mat image_proxy; - resize(image_gray, image_proxy, Size(cvRound(image.cols * scale), cvRound(image.rows * scale)), 0, 0, DOWNSCALE_MODE); - image_proxy.copyTo(image_gray); - myBoundingBox.x *= scale; - myBoundingBox.y *= scale; - myBoundingBox.width *= scale; - myBoundingBox.height *= scale; - } - model = Ptr(new TrackerTLDModel(params, image_gray, myBoundingBox, data->getMinSize())); - detector = Ptr(new TLDDetector(params, model)); - data->confident = false; - data->failedLastTime = false; - - return true; -} - -bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) -{ - Mat image_gray, image_blurred, imageForDetector; - cvtColor( image, image_gray, COLOR_BGR2GRAY ); - double scale = data->getScale(); - if( scale > 1.0 ) - resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, DOWNSCALE_MODE); - else - imageForDetector = image_gray; - GaussianBlur(imageForDetector, image_blurred, GaussBlurKernelSize, 0.0); - TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(model)); - data->frameNum++; - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - std::vector detectorResults; - //best overlap around 92% - - std::vector candidates; - std::vector candidatesRes; - bool trackerNeedsReInit = false; - for( int i = 0; i < 2; i++ ) - { - Rect2d tmpCandid = boundingBox; - if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || - ( (i == 1) && detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults) ) ) - { - candidates.push_back(tmpCandid); - if( i == 0 ) - resample(image_gray, tmpCandid, standardPatch); - else - resample(imageForDetector, tmpCandid, standardPatch); - candidatesRes.push_back(tldModel->Sc(standardPatch)); - } - else - { - if( i == 0 ) - trackerNeedsReInit = true; - } - } - - std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); - - //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); - //for( int i = 0; i < (int)candidatesRes.size(); i++ ) - //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); - //data->printme(); - //tldModel->printme(stdout); - - if( it == candidatesRes.end() ) - { - data->confident = false; - data->failedLastTime = true; - return false; - } - else - { - boundingBox = candidates[it - candidatesRes.begin()]; - data->failedLastTime = false; - if( trackerNeedsReInit || it != candidatesRes.begin() ) - trackerProxy->init(image, boundingBox); - } - -#if 1 - if( it != candidatesRes.end() ) - { - resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); - //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); - //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) - //dfprintf((stderr, "detector WON\n")); - } - else - { - //dfprintf((stderr, "%d x x\n", data->frameNum)); - } -#endif - - if( *it > CORE_THRESHOLD ) - data->confident = true; - - if( data->confident ) - { - Pexpert pExpert(imageForDetector, image_blurred, boundingBox, detector, params, data->getMinSize()); - Nexpert nExpert(imageForDetector, boundingBox, detector, params); - std::vector > examplesForModel, examplesForEnsemble; - examplesForModel.reserve(100); examplesForEnsemble.reserve(100); - int negRelabeled = 0; - for( int i = 0; i < (int)detectorResults.size(); i++ ) - { - bool expertResult; - if( detectorResults[i].isObject ) - { - expertResult = nExpert(detectorResults[i].rect); - if( expertResult != detectorResults[i].isObject ) - negRelabeled++; - } - else - { - expertResult = pExpert(detectorResults[i].rect); - } - - detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); - detectorResults[i].isObject = expertResult; - } - tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); - //dprintf(("%d relabeled by nExpert\n", negRelabeled)); - pExpert.additionalExamples(examplesForModel, examplesForEnsemble); - tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); - examplesForModel.clear(); examplesForEnsemble.clear(); - nExpert.additionalExamples(examplesForModel, examplesForEnsemble); - tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); - } - else - { -#ifdef CLOSED_LOOP - tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); -#endif - } - - return true; -} - -TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize):minSize_(minSize), -timeStampPositiveNext(0), timeStampNegativeNext(0), params_(params), boundingBox_(boundingBox) -{ - originalVariance_ = variance(image(boundingBox)); - std::vector closest, scanGrid; - Mat scaledImg, blurredImg, image_blurred; - - double scale = scaleAndBlur(image, cvRound(log(1.0 * boundingBox.width / (minSize.width)) / log(SCALE_STEP)), - scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); - GaussianBlur(image, image_blurred, GaussBlurKernelSize, 0.0); - TLDDetector::generateScanGrid(image.rows, image.cols, minSize, scanGrid); - getClosestN(scanGrid, Rect2d(boundingBox.x / scale, boundingBox.y / scale, boundingBox.width / scale, boundingBox.height / scale), 10, closest); - - Mat_ blurredPatch(minSize); - TLDEnsembleClassifier::makeClassifiers(minSize, MEASURES_PER_CLASSIFIER, GRIDSIZE, classifiers); - - positiveExamples.reserve(200); - for( int i = 0; i < (int)closest.size(); i++ ) - { - for( int j = 0; j < 20; j++ ) - { - Point2f center; - Size2f size; - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); - center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); - size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); - size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); - float angle = (float)rng.uniform(-10.0, 10.0); - - resample(scaledImg, RotatedRect(center, size, angle), standardPatch); - - for( int y = 0; y < standardPatch.rows; y++ ) - { - for( int x = 0; x < standardPatch.cols; x++ ) - { - standardPatch(x, y) += (uchar)rng.gaussian(5.0); - } - } - -#ifdef BLUR_AS_VADIM - GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); - resize(blurredPatch, blurredPatch, minSize); -#else - resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); -#endif - pushIntoModel(standardPatch, true); - for( int k = 0; k < (int)classifiers.size(); k++ ) - classifiers[k].integrate(blurredPatch, true); - } - } - - TLDDetector::generateScanGrid(image.rows, image.cols, minSize, scanGrid, true); - negativeExamples.clear(); - negativeExamples.reserve(NEG_EXAMPLES_IN_INIT_MODEL); - std::vector indices; - indices.reserve(NEG_EXAMPLES_IN_INIT_MODEL); - while( (int)negativeExamples.size() < NEG_EXAMPLES_IN_INIT_MODEL ) - { - int i = rng.uniform((int)0, (int)scanGrid.size()); - if( std::find(indices.begin(), indices.end(), i) == indices.end() && overlap(boundingBox, scanGrid[i]) < NEXPERT_THRESHOLD ) - { - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - resample(image, scanGrid[i], standardPatch); - pushIntoModel(standardPatch, false); - - resample(image_blurred, scanGrid[i], blurredPatch); - for( int k = 0; k < (int)classifiers.size(); k++ ) - classifiers[k].integrate(blurredPatch, false); - } - } - //dprintf(("positive patches: %d\nnegative patches: %d\n", (int)positiveExamples.size(), (int)negativeExamples.size())); -} - -void TLDDetector::generateScanGrid(int rows, int cols, Size initBox, std::vector& res, bool withScaling) -{ - res.clear(); - //scales step: SCALE_STEP; hor step: 10% of width; verstep: 10% of height; minsize: 20pix - for( double h = initBox.height, w = initBox.width; h < cols && w < rows; ) - { - for( double x = 0; (x + w + 1.0) <= cols; x += (0.1 * w) ) - { - for( double y = 0; (y + h + 1.0) <= rows; y += (0.1 * h) ) - res.push_back(Rect2d(x, y, w, h)); - } - if( withScaling ) - { - if( h <= initBox.height ) - { - h /= SCALE_STEP; w /= SCALE_STEP; - if( h < 20 || w < 20 ) - { - h = initBox.height * SCALE_STEP; w = initBox.width * SCALE_STEP; - CV_Assert( h > initBox.height || w > initBox.width); - } - } - else - { - h *= SCALE_STEP; w *= SCALE_STEP; - } - } - else - { - break; - } - } - //dprintf(("%d rects in res\n", (int)res.size())); -} - -bool TLDDetector::detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches) -{ - TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(model)); - Size initSize = tldModel->getMinSize(); - patches.clear(); - - Mat resized_img, blurred_img; - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - img.copyTo(resized_img); - imgBlurred.copyTo(blurred_img); - double originalVariance = tldModel->getOriginalVariance(); ; - int dx = initSize.width / 10, dy = initSize.height / 10; - Size2d size = img.size(); - double scale = 1.0; - int total = 0, pass = 0; - int npos = 0, nneg = 0; - double tmp = 0, maxSc = -5.0; - Rect2d maxScRect; - - //START_TICK("detector"); - do - { - Mat_ intImgP, intImgP2; - computeIntegralImages(resized_img, intImgP, intImgP2); - - tldModel->prepareClassifiers((int)blurred_img.step[0]); - for( int i = 0, imax = cvFloor((0.0 + resized_img.cols - initSize.width) / dx); i < imax; i++ ) - { - for( int j = 0, jmax = cvFloor((0.0 + resized_img.rows - initSize.height) / dy); j < jmax; j++ ) - { - LabeledPatch labPatch; - total++; - if( !patchVariance(intImgP, intImgP2, originalVariance, Point(dx * i, dy * j), initSize) ) - continue; - if( tldModel->ensembleClassifierNum(&blurred_img.at(dy * j, dx * i)) <= ENSEMBLE_THRESHOLD ) - continue; - pass++; - - labPatch.rect = Rect2d(dx * i * scale, dy * j * scale, initSize.width * scale, initSize.height * scale); - resample(resized_img, Rect2d(Point(dx * i, dy * j), initSize), standardPatch); - tmp = tldModel->Sr(standardPatch); - labPatch.isObject = tmp > THETA_NN; - labPatch.shouldBeIntegrated = abs(tmp - THETA_NN) < 0.1; - patches.push_back(labPatch); - - if( !labPatch.isObject ) - { - nneg++; - continue; - } - else - { - npos++; - } - tmp = tldModel->Sc(standardPatch); - if( tmp > maxSc ) - { - maxSc = tmp; - maxScRect = labPatch.rect; - } - } - } - - size.width /= SCALE_STEP; - size.height /= SCALE_STEP; - scale *= SCALE_STEP; - resize(img, resized_img, size, 0, 0, DOWNSCALE_MODE); - GaussianBlur(resized_img, blurred_img, GaussBlurKernelSize, 0.0f); - } - while( size.width >= initSize.width && size.height >= initSize.height ); - //END_TICK("detector"); - - //dfprintf((stdout, "after NCC: nneg = %d npos = %d\n", nneg, npos)); -#if !1 - std::vector poss, negs; - - for( int i = 0; i < (int)patches.size(); i++ ) - { - if( patches[i].isObject ) - poss.push_back(patches[i].rect); - else - negs.push_back(patches[i].rect); - } - //dfprintf((stdout, "%d pos and %d neg\n", (int)poss.size(), (int)negs.size())); - drawWithRects(img, negs, poss, "tech"); -#endif - - //dfprintf((stdout, "%d after ensemble\n", pass)); - if( maxSc < 0 ) - return false; - res = maxScRect; - return true; -} - -/** Computes the variance of subimage given by box, with the help of two integral - * images intImgP and intImgP2 (sum of squares), which should be also provided.*/ -bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size) -{ - int x = (pt.x), y = (pt.y), width = (size.width), height = (size.height); - CV_Assert( 0 <= x && (x + width) < intImgP.cols && (x + width) < intImgP2.cols ); - CV_Assert( 0 <= y && (y + height) < intImgP.rows && (y + height) < intImgP2.rows ); - double p = 0, p2 = 0; - double A, B, C, D; - - A = intImgP(y, x); - B = intImgP(y, x + width); - C = intImgP(y + height, x); - D = intImgP(y + height, x + width); - p = (A + D - B - C) / (width * height); - - A = intImgP2(y, x); - B = intImgP2(y, x + width); - C = intImgP2(y + height, x); - D = intImgP2(y + height, x + width); - p2 = (A + D - B - C) / (width * height); - - return ((p2 - p * p) > VARIANCE_THRESHOLD * originalVariance); -} - -double TrackerTLDModel::ensembleClassifierNum(const uchar* data) -{ - double p = 0; - for( int k = 0; k < (int)classifiers.size(); k++ ) - p += classifiers[k].posteriorProbabilityFast(data); - p /= classifiers.size(); - return p; -} - -double TrackerTLDModel::Sr(const Mat_& patch) -{ - double splus = 0.0, sminus = 0.0; - for( int i = 0; i < (int)positiveExamples.size(); i++ ) - splus = std::max(splus, 0.5 * (NCC(positiveExamples[i], patch) + 1.0)); - for( int i = 0; i < (int)negativeExamples.size(); i++ ) - sminus = std::max(sminus, 0.5 * (NCC(negativeExamples[i], patch) + 1.0)); - if( splus + sminus == 0.0) - return 0.0; - return splus / (sminus + splus); -} - -double TrackerTLDModel::Sc(const Mat_& patch) -{ - double splus = 0.0, sminus = 0.0; - int med = getMedian(timeStampsPositive); - for( int i = 0; i < (int)positiveExamples.size(); i++ ) - { - if( (int)timeStampsPositive[i] <= med ) - splus = std::max(splus, 0.5 * (NCC(positiveExamples[i], patch) + 1.0)); - } - for( int i = 0; i < (int)negativeExamples.size(); i++ ) - sminus = std::max(sminus, 0.5 * (NCC(negativeExamples[i], patch) + 1.0)); - if( splus + sminus == 0.0 ) - return 0.0; - return splus / (sminus + splus); -} - -void TrackerTLDModel::integrateRelabeled(Mat& img, Mat& imgBlurred, const std::vector& patches) -{ - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(minSize_); - int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; - for( int k = 0; k < (int)patches.size(); k++ ) - { - if( patches[k].shouldBeIntegrated ) - { - resample(img, patches[k].rect, standardPatch); - if( patches[k].isObject ) - { - positiveIntoModel++; - pushIntoModel(standardPatch, true); - } - else - { - negativeIntoModel++; - pushIntoModel(standardPatch, false); - } - } - -#ifdef CLOSED_LOOP - if( patches[k].shouldBeIntegrated || !patches[k].isPositive ) -#else - if( patches[k].shouldBeIntegrated ) -#endif - { - resample(imgBlurred, patches[k].rect, blurredPatch); - if( patches[k].isObject ) - positiveIntoEnsemble++; - else - negativeIntoEnsemble++; - for( int i = 0; i < (int)classifiers.size(); i++ ) - classifiers[i].integrate(blurredPatch, patches[k].isObject); - } - } - /* - if( negativeIntoModel > 0 ) - dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); - if( positiveIntoModel > 0) - dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); - if( negativeIntoEnsemble > 0 ) - dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); - if( positiveIntoEnsemble > 0 ) - dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); - dfprintf((stdout, "\n"));*/ -} - -void TrackerTLDModel::integrateAdditional(const std::vector >& eForModel, const std::vector >& eForEnsemble, bool isPositive) -{ - int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; - for( int k = 0; k < (int)eForModel.size(); k++ ) - { - double sr = Sr(eForModel[k]); - if( ( sr > THETA_NN ) != isPositive ) - { - if( isPositive ) - { - positiveIntoModel++; - pushIntoModel(eForModel[k], true); - } - else - { - negativeIntoModel++; - pushIntoModel(eForModel[k], false); - } - } - double p = 0; - for( int i = 0; i < (int)classifiers.size(); i++ ) - p += classifiers[i].posteriorProbability(eForEnsemble[k].data, (int)eForEnsemble[k].step[0]); - p /= classifiers.size(); - if( ( p > ENSEMBLE_THRESHOLD ) != isPositive ) - { - if( isPositive ) - positiveIntoEnsemble++; - else - negativeIntoEnsemble++; - for( int i = 0; i < (int)classifiers.size(); i++ ) - classifiers[i].integrate(eForEnsemble[k], isPositive); - } - } - /* - if( negativeIntoModel > 0 ) - dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); - if( positiveIntoModel > 0 ) - dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); - if( negativeIntoEnsemble > 0 ) - dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); - if( positiveIntoEnsemble > 0 ) - dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); - dfprintf((stdout, "\n"));*/ -} - -int TrackerTLDImpl::Pexpert::additionalExamples(std::vector >& examplesForModel, std::vector >& examplesForEnsemble) -{ - examplesForModel.clear(); examplesForEnsemble.clear(); - examplesForModel.reserve(100); examplesForEnsemble.reserve(100); - - std::vector closest, scanGrid; - Mat scaledImg, blurredImg; - - double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)), - scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); - TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid); - getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest); - - for( int i = 0; i < (int)closest.size(); i++ ) - { - for( int j = 0; j < 10; j++ ) - { - Point2f center; - Size2f size; - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(initSize_); - center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); - center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); - size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); - size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); - float angle = (float)rng.uniform(-5.0, 5.0); - - for( int y = 0; y < standardPatch.rows; y++ ) - { - for( int x = 0; x < standardPatch.cols; x++ ) - { - standardPatch(x, y) += (uchar)rng.gaussian(5.0); - } - } -#ifdef BLUR_AS_VADIM - GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); - resize(blurredPatch, blurredPatch, initSize_); -#else - resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); -#endif - resample(scaledImg, RotatedRect(center, size, angle), standardPatch); - examplesForModel.push_back(standardPatch); - examplesForEnsemble.push_back(blurredPatch); - } - } - return 0; -} - -bool TrackerTLDImpl::Nexpert::operator()(Rect2d box) -{ - if( overlap(resultBox_, box) < NEXPERT_THRESHOLD ) - return false; - else - return true; -} - -Data::Data(Rect2d initBox) -{ - double minDim = std::min(initBox.width, initBox.height); - scale = 20.0 / minDim; - minSize.width = (int)(initBox.width * 20.0 / minDim); - minSize.height = (int)(initBox.height * 20.0 / minDim); - frameNum = 0; - //dprintf(("minSize = %dx%d\n", minSize.width, minSize.height)); -} - -void Data::printme(FILE* port) -{ - dfprintf((port, "Data:\n")); - dfprintf((port, "\tframeNum = %d\n", frameNum)); - dfprintf((port, "\tconfident = %s\n", confident?"true":"false")); - dfprintf((port, "\tfailedLastTime = %s\n", failedLastTime?"true":"false")); - dfprintf((port, "\tminSize = %dx%d\n", minSize.width, minSize.height)); -} - -void TrackerTLDModel::printme(FILE* port) -{ - dfprintf((port, "TrackerTLDModel:\n")); - dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); - dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); -} - -void MyMouseCallbackDEBUG::onMouse(int event, int x, int y) -{ - if( event == EVENT_LBUTTONDOWN ) - { - Mat imgCanvas; - img_.copyTo(imgCanvas); - TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(detector_->model)); - Size initSize = tldModel->getMinSize(); - Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - double originalVariance = tldModel->getOriginalVariance(); - double tmp; - - Mat resized_img, blurred_img; - double scale = SCALE_STEP; - //double scale = SCALE_STEP * SCALE_STEP * SCALE_STEP * SCALE_STEP; - Size2d size(img_.cols / scale, img_.rows / scale); - resize(img_, resized_img, size); - resize(imgBlurred_, blurred_img, size); - - Mat_ intImgP, intImgP2; - detector_->computeIntegralImages(resized_img, intImgP, intImgP2); - - int dx = initSize.width / 10, dy = initSize.height / 10, - i = (int)(x / scale / dx), j = (int)(y / scale / dy); - - dfprintf((stderr, "patchVariance = %s\n", (detector_->patchVariance(intImgP, intImgP2, originalVariance, - Point(dx * i, dy * j), initSize))?"true":"false")); - tldModel->prepareClassifiers((int)blurred_img.step[0]); - dfprintf((stderr, "p = %f\n", (tldModel->ensembleClassifierNum(&blurred_img.at(dy * j, dx * i))))); - fprintf(stderr, "ensembleClassifier = %s\n", - (!(tldModel->ensembleClassifierNum(&blurred_img.at(dy * j, dx * i)) > ENSEMBLE_THRESHOLD))?"true":"false"); - - resample(resized_img, Rect2d(Point(dx * i, dy * j), initSize), standardPatch); - tmp = tldModel->Sr(standardPatch); - dfprintf((stderr, "Sr = %f\n", tmp)); - dfprintf((stderr, "isObject = %s\n", (tmp > THETA_NN)?"true":"false")); - dfprintf((stderr, "shouldBeIntegrated = %s\n", (abs(tmp - THETA_NN) < 0.1)?"true":"false")); - dfprintf((stderr, "Sc = %f\n", tldModel->Sc(standardPatch))); - - rectangle(imgCanvas, Rect2d(Point2d(scale * dx * i, scale * dy * j), Size2d(initSize.width * scale, initSize.height * scale)), 0, 2, 1 ); - imshow("picker", imgCanvas); - waitKey(); - } -} - -void TrackerTLDModel::pushIntoModel(const Mat_& example, bool positive) -{ - std::vector >* proxyV; - int* proxyN; - std::vector* proxyT; - if( positive ) - { - proxyV = &positiveExamples; - proxyN = &timeStampPositiveNext; - proxyT = &timeStampsPositive; - } - else - { - proxyV = &negativeExamples; - proxyN = &timeStampNegativeNext; - proxyT = &timeStampsNegative; - } - if( (int)proxyV->size() < MAX_EXAMPLES_IN_MODEL ) - { - proxyV->push_back(example); - proxyT->push_back(*proxyN); - } - else - { - int index = rng.uniform((int)0, (int)proxyV->size()); - (*proxyV)[index] = example; - (*proxyT)[index] = (*proxyN); - } - (*proxyN)++; -} -void TrackerTLDModel::prepareClassifiers(int rowstep) -{ - for( int i = 0; i < (int)classifiers.size(); i++ ) - classifiers[i].prepareClassifier(rowstep); -} - -} /* namespace tld */ - -} /* namespace cv */ diff --git a/modules/tracking/src/tld_tracker.hpp b/modules/tracking/src/tld_tracker.hpp deleted file mode 100644 index 39f983244..000000000 --- a/modules/tracking/src/tld_tracker.hpp +++ /dev/null @@ -1,125 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc.hpp" -#include -#include - -namespace cv {namespace tld -{ - -//debug functions and variables -#define ALEX_DEBUG -#ifdef ALEX_DEBUG -#define dfprintf(x) fprintf x -#define dprintf(x) printf x -#else -#define dfprintf(x) -#define dprintf(x) -#endif -#define MEASURE_TIME(a)\ -{\ - clock_t start; float milisec = 0.0; \ - start = clock(); {a} milisec = 1000.0 * (clock() - start) / CLOCKS_PER_SEC; \ - dprintf(("%-90s took %f milis\n", #a, milisec));\ -} -#define HERE dprintf(("line %d\n", __LINE__)); fflush(stderr); -#define START_TICK(name)\ -{ \ - clock_t start; double milisec = 0.0; start = clock(); -#define END_TICK(name) milisec = 1000.0 * (clock() - start) / CLOCKS_PER_SEC; \ - dprintf(("%s took %f milis\n", name, milisec)); \ -} -extern Rect2d etalon; -void myassert(const Mat& img); -void printPatch(const Mat_& standardPatch); -std::string type2str(const Mat& mat); -void drawWithRects(const Mat& img, std::vector& blackOnes, Rect2d whiteOne = Rect2d(-1.0, -1.0, -1.0, -1.0)); -void drawWithRects(const Mat& img, std::vector& blackOnes, std::vector& whiteOnes, String fileName = ""); - -//aux functions and variables -template inline T CLIP(T x, T a, T b){ return std::min(std::max(x, a), b); } -/** Computes overlap between the two given rectangles. Overlap is computed as ratio of rectangles' intersection to that - * of their union.*/ -double overlap(const Rect2d& r1, const Rect2d& r2); -/** Resamples the area surrounded by r2 in img so it matches the size of samples, where it is written.*/ -void resample(const Mat& img, const RotatedRect& r2, Mat_& samples); -/** Specialization of resample() for rectangles without retation for better performance and simplicity.*/ -void resample(const Mat& img, const Rect2d& r2, Mat_& samples); -/** Computes the variance of single given image.*/ -double variance(const Mat& img); -/** Computes normalized corellation coefficient between the two patches (they should be - * of the same size).*/ -double NCC(const Mat_& patch1, const Mat_& patch2); -void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res); -double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep); -int getMedian(const std::vector& values, int size = -1); - -class TLDEnsembleClassifier -{ -public: - static int makeClassifiers(Size size, int measurePerClassifier, int gridSize, std::vector& classifiers); - void integrate(const Mat_& patch, bool isPositive); - double posteriorProbability(const uchar* data, int rowstep) const; - double posteriorProbabilityFast(const uchar* data) const; - void prepareClassifier(int rowstep); -private: - TLDEnsembleClassifier(const std::vector& meas, int beg, int end); - static void stepPrefSuff(std::vector & arr, int pos, int len, int gridSize); - int code(const uchar* data, int rowstep) const; - int codeFast(const uchar* data) const; - std::vector posAndNeg; - std::vector measurements; - std::vector offset; - int lastStep_; -}; - -class TrackerProxy -{ -public: - virtual bool init(const Mat& image, const Rect2d& boundingBox) = 0; - virtual bool update(const Mat& image, Rect2d& boundingBox) = 0; - virtual ~TrackerProxy(){} -}; - -}} From f77a46167fb182a506e68b6a94d7fd7b9457f7d9 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Mon, 8 Jun 2015 00:45:03 +0900 Subject: [PATCH 14/98] Updated the support for color-names features and fixing some typos --- .../include/opencv2/tracking/tracker.hpp | 4 +- modules/tracking/src/trackerKCF.cpp | 109 ++++++++++++------ 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 2435b6510..c4bce6993 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1189,8 +1189,8 @@ class CV_EXPORTS_W TrackerTLD : public Tracker BOILERPLATE_CODE("TLD",TrackerTLD); }; -/** @brief KCF is a novel tracking framework that utilize properties of circulant matrix to enhance the processing speed. - * This tracking method is implementation of @cite KCF_ECCV which is extended to KFC with color-names features (@cite KCF_CN). +/** @brief KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. + * This tracking method is an implementation of @cite KCF_ECCV which is extended to KFC with color-names features (@cite KCF_CN). * The original paper of KCF is available at * as well as the matlab implementation. For more information about KCF with color-names features, please refer to * . diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index c76c15a50..73d645787 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -88,8 +88,11 @@ namespace cv{ * KCF functions and vars */ void createHanningWindow(OutputArray _dst, const cv::Size winSize, const int type) const; + void inline fft2(const Mat src, std::vector & dest) const; void inline fft2(const Mat src, Mat & dest) const; - void inline ifft2(const Mat src, Mat & dest) const ; + void inline ifft2(const Mat src, Mat & dest) const ; + void inline pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB=false) const; + void inline sumChannels(std::vector src, Mat & dest) const; bool getSubWindow(const Mat img, const Rect roi, Mat& patch) const; void extractCN(Mat _patch, Mat & cnFeatures) const; void denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k) const; @@ -127,6 +130,8 @@ namespace cv{ { isInit = false; resizeImage = false; + + CV_Assert(params.descriptor == GRAY || params.descriptor == CN /*|| params.descriptor == CN2*/); } void TrackerKCFImpl::read( const cv::FileNode& fn ){ @@ -163,7 +168,7 @@ namespace cv{ // add padding to the roi roi.x-=roi.width/2; - roi.y-=roi.height/2+1; + roi.y-=roi.height/2; roi.width*=2; roi.height*=2; @@ -173,7 +178,6 @@ namespace cv{ Mat layers[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; merge(layers, 10, hann); } - if(params.descriptor != GRAY){printf("The choosen descriptor mode is not available! Please use GRAY descriptor, other descriptors will be avaiable soon.\n");return false;}//temporary, will be updated soon // create gaussian response y=Mat::zeros((int)roi.height,(int)roi.width,CV_64F); @@ -229,18 +233,19 @@ namespace cv{ // Kernel Regularized Least-Squares, calculate alphas denseGaussKernel(params.sigma,x,x,k); + fft2(k,kf); kf=kf+params.lambda; /* TODO: optimize this element-wise division * new_alphaf=yf./kf - * z=[(ax+bd)+i(bc-ad)]/(c^2+d^2) + * z=(a+bi)/(c+di)[(ac+bd)+i(bc-ad)]/(c^2+d^2) */ new_alphaf=Mat_(yf.rows, yf.cols); std::complex temp; for(int i=0;i(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf.at(i,j)[0],kf.at(i,j)[1])/*+complex(0.0000000001,0.0000000001)*/); + temp=std::complex(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf.at(i,j)[0],kf.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); new_alphaf.at(i,j)[0]=temp.real(); new_alphaf.at(i,j)[1]=temp.imag(); } @@ -250,7 +255,7 @@ namespace cv{ new_z=x.clone(); if(frame==0){ alphaf=new_alphaf.clone(); - z=x; + z=x.clone(); }else{ alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; @@ -309,21 +314,62 @@ namespace cv{ } /* - * simplification of fourier transoform function in opencv + * simplification of fourier transform function in opencv */ void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest)const { - Mat planes[] = {Mat_(src), Mat::zeros(src.size(), CV_64F)}; - merge(planes, 2, dest); - dft(dest,dest,DFT_COMPLEX_OUTPUT); + std::vector layers(src.channels()); + std::vector outputs(src.channels()); + + split(src, layers); + + for(int i=0;i & dest) const{ + std::vector layers(src.channels()); + dest.clear(); + dest.resize(src.channels()); + + split(src, layers); + + for(int i=0;i src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const{ + dest.clear(); + dest.resize(src1.size()); + + for(unsigned i=0;i src, Mat & dest) const{ + dest=src[0].clone(); + for(unsigned i=1;iimg.cols?_roi.width+_roi.x-img.cols:0); copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); - + if(patch.rows==0 || patch.cols==0)return false; + // extract the desired descriptors switch(params.descriptor){ case GRAY: if(img.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); + patch.convertTo(patch,CV_64F); + patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 break; case CN: CV_Assert(img.channels() == 3); extractCN(patch,patch); break; - default: + case CN2: if(patch.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); break; } - - patch.convertTo(patch,CV_64F); - - patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 + patch=patch.mul(hann); // hann window filter return true; @@ -407,25 +453,26 @@ namespace cv{ * dense gauss kernel function */ void TrackerKCFImpl::denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k)const{ - Mat _xf, _yf, xyf,xy; + std::vector _xf,_yf,xyf_v; + Mat xy,xyf; double normX, normY; fft2(_x,_xf); fft2(_y,_yf); + normX=norm(_x); normX*=normX; normY=norm(_y); normY*=normY; - mulSpectrums(_xf,_yf,xyf,0,true); - + pixelWiseMult(_xf,_yf,xyf_v,0,true); + sumChannels(xyf_v,xyf); ifft2(xyf,xyf); - shiftRows(xyf, _x.rows/2); - shiftCols(xyf,_x.cols/2); +// shiftRows(xyf, _x.rows/2); +// shiftCols(xyf, _x.cols/2); //(xx + yy - 2 * xy) / numel(x) - xy=(normX+normY-2*xyf)/(_x.rows*_x.cols); - + xy=(normX+normY-2*xyf)/(_x.rows*_x.cols*_x.channels()); // TODO: check wether we really need thresholding or not //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) @@ -441,7 +488,7 @@ namespace cv{ } - /* CIRCULAR SHIT Function + /* CIRCULAR SHIFT Function * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv */ // circular shift one row from up to down @@ -523,16 +570,12 @@ namespace cv{ interp_factor=0.075; output_sigma_factor=1.0/16.0; resize=true; - max_patch_size=80*80; - descriptor=GRAY; + max_patch_size=100*100; + descriptor=CN; } - void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){ - - } + void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){} - void TrackerKCF::Params::write( cv::FileStorage& /*fs*/ ) const{ - - } + void TrackerKCF::Params::write( cv::FileStorage& /*fs*/ ) const{} } /* namespace cv */ From 50492e9e3804232eb3f47860fd649a447e18c137 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Mon, 8 Jun 2015 10:42:19 +0900 Subject: [PATCH 15/98] Fixing the tabulation --- .../tracking/include/opencv2/tracking/tracker.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index c4bce6993..e43dbd45b 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1205,13 +1205,13 @@ class CV_EXPORTS_W TrackerKCF : public Tracker void read( const FileNode& /*fn*/ ); void write( FileStorage& /*fs*/ ) const; - double sigma; // gaussian kernel bandwidth - double lambda; // regularization - double interp_factor; // linear interpolation factor for adaptation - double output_sigma_factor; // spatial bandwidth (proportional to target) - bool resize; // activate the resize feature to improve the processing speed - int max_patch_size; // threshold for the ROI size - MODE descriptor; // descriptor type + double sigma; // gaussian kernel bandwidth + double lambda; // regularization + double interp_factor; // linear interpolation factor for adaptation + double output_sigma_factor; // spatial bandwidth (proportional to target) + bool resize; // activate the resize feature to improve the processing speed + int max_patch_size; // threshold for the ROI size + MODE descriptor; // descriptor type }; /** @brief Constructor From 135cf2960df98b3b1ad7c188734b39988863ad4f Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 16:06:52 +0900 Subject: [PATCH 16/98] Added test on TLD Dataset --- modules/tracking/test/test_tldDataset.cpp | 227 ++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 modules/tracking/test/test_tldDataset.cpp diff --git a/modules/tracking/test/test_tldDataset.cpp b/modules/tracking/test/test_tldDataset.cpp new file mode 100644 index 000000000..63cb378c1 --- /dev/null +++ b/modules/tracking/test/test_tldDataset.cpp @@ -0,0 +1,227 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include +#include +#include + +#include + +using namespace std; +using namespace cv; + +#define NUM_TEST_FRAMES 500 +#define TEST_VIDEO_INDEX 1 //TLD Dataset Video Index from 1-10 +//#define RECORD_VIDEO_FLG + +static Mat image; +static Rect2d boundingBox; +static bool paused; +static bool selectObject = false; +static bool startSelection = false; + +static void onMouse(int event, int x, int y, int, void*) +{ + if (!selectObject) + { + switch (event) + { + case EVENT_LBUTTONDOWN: + //set origin of the bounding box + startSelection = true; + boundingBox.x = x; + boundingBox.y = y; + boundingBox.width = boundingBox.height = 0; + break; + case EVENT_LBUTTONUP: + //sei with and height of the bounding box + boundingBox.width = std::abs(x - boundingBox.x); + boundingBox.height = std::abs(y - boundingBox.y); + paused = false; + selectObject = true; + break; + case EVENT_MOUSEMOVE: + + if (startSelection && !selectObject) + { + //draw the bounding box + Mat currentFrame; + image.copyTo(currentFrame); + rectangle(currentFrame, Point((int)boundingBox.x, (int)boundingBox.y), Point(x, y), Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", currentFrame); + } + break; + } + } +} + +void main() +{ + // + // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" + // + string tracker_algorithm_name = "TLD"; + + Mat frame; + paused = false; + namedWindow("Tracking API", 0); + setMouseCallback("Tracking API", onMouse, 0); + + Ptr tracker = Tracker::create(tracker_algorithm_name); + if (tracker == NULL) + { + cout << "***Error in the instantiation of the tracker...***\n"; + getchar(); + return; + } + + //Get the first frame + ////Open the capture + // VideoCapture cap(0); + // if( !cap.isOpened() ) + // { + // cout << "Video stream error"; + // return; + // } + //cap >> frame; + + //From TLD dataset + selectObject = true; + boundingBox = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/TLD_dataset"); + + frame = tld::tld_getNextDatasetFrame(); + frame.copyTo(image); + + // Setup output video +#ifdef RECORD_VIDEO_FLG + String outputFilename = "test.avi"; + VideoWriter outputVideo; + outputVideo.open(outputFilename, -1, 30, Size(image.cols, image.rows)); + + if (!outputVideo.isOpened()) + { + std::cout << "!!! Output video could not be opened" << std::endl; + getchar(); + return; + } +#endif + + rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", image); + + + bool initialized = false; + int frameCounter = 0; + + //Time measurment + float e3 = getTickCount(); + + for (;;) + { + //Time measurment + float e1 = getTickCount(); + //Frame num + frameCounter++; + if (frameCounter == NUM_TEST_FRAMES) break; + + char c = (char)waitKey(2); + if (c == 'q' || c == 27) + break; + if (c == 'p') + paused = !paused; + + if (!paused) + { + //cap >> frame; + frame = tld::tld_getNextDatasetFrame(); + if (frame.empty()) + { + break; + } + frame.copyTo(image); + + if (selectObject) + { + if (!initialized) + { + //initializes the tracker + if (!tracker->init(frame, boundingBox)) + { + cout << "***Could not initialize tracker...***\n"; + return; + } + initialized = true; + rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); + } + else + { + //updates the tracker + if (tracker->update(frame, boundingBox)) + { + rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); + } + } + } + imshow("Tracking API", image); + +#ifdef RECORD_VIDEO_FLG + outputVideo << image; +#endif + + + //Time measurment + float e2 = getTickCount(); + float t1 = (e2 - e1) / getTickFrequency(); + cout << frameCounter << "\tframe : " << t1 * 1000 << "ms" << endl; + + //waitKey(0); + } + } + + //Time measurment + float e4 = getTickCount(); + float t2 = (e4 - e3) / getTickFrequency(); + cout << "Average Time for Frame: " << t2 * 1000 / frameCounter << "ms" << endl; + cout << "Average FPS: " << 1.0 / t2*frameCounter << endl; + + + waitKey(0); +} \ No newline at end of file From 98afd5525e340f1909379f1eea0cd5be7807c543 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 21:21:27 +0900 Subject: [PATCH 17/98] Added BSD-compatible license Added BSD-compatible license to some files --- .../include/opencv2/tracking/tldDataset.hpp | 41 +++++++++++++++++++ modules/tracking/src/tldDataset.cpp | 41 +++++++++++++++++++ modules/tracking/src/tldDetector.cpp | 41 +++++++++++++++++++ modules/tracking/src/tldDetector.hpp | 41 +++++++++++++++++++ .../tracking/src/tldEnsembleClassifier.cpp | 41 +++++++++++++++++++ .../tracking/src/tldEnsembleClassifier.hpp | 41 +++++++++++++++++++ modules/tracking/src/tldModel.cpp | 41 +++++++++++++++++++ modules/tracking/src/tldModel.hpp | 41 +++++++++++++++++++ modules/tracking/src/tldTracker.hpp | 1 + 9 files changed, 329 insertions(+) diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index 0505562b1..00d8dd711 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #ifndef OPENCV_TLD_DATASET #define OPENCV_TLD_DATASET diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 7f9b9e5d0..2f276306a 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "opencv2/tracking/tldDataset.hpp" namespace cv diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 2caf179ec..8a11ecdff 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "tldDetector.hpp" namespace cv diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index feead4826..b51108e23 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #ifndef OPENCV_TLD_DETECTOR #define OPENCV_TLD_DETECTOR diff --git a/modules/tracking/src/tldEnsembleClassifier.cpp b/modules/tracking/src/tldEnsembleClassifier.cpp index 3912d55ab..32286928e 100644 --- a/modules/tracking/src/tldEnsembleClassifier.cpp +++ b/modules/tracking/src/tldEnsembleClassifier.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "tldEnsembleClassifier.hpp" namespace cv diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp index 446a0359c..2725700e6 100644 --- a/modules/tracking/src/tldEnsembleClassifier.hpp +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include #include "precomp.hpp" diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 212b42d41..4b0e07100 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "tldModel.hpp" namespace cv diff --git a/modules/tracking/src/tldModel.hpp b/modules/tracking/src/tldModel.hpp index a0ac0a349..3a1e5149a 100644 --- a/modules/tracking/src/tldModel.hpp +++ b/modules/tracking/src/tldModel.hpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #ifndef OPENCV_TLD_MODEL #define OPENCV_TLD_MODEL diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 1cc9e4863..de2622383 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -38,6 +38,7 @@ // the use of this software, even if advised of the possibility of such damage. // //M*/ + #ifndef OPENCV_TLD_TRACKER #define OPENCV_TLD_TRACKER From 85fdd132203e4769e376c00410cbb23539c2f4c8 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 22:54:51 +0900 Subject: [PATCH 18/98] Fixed header --- modules/tracking/test/test_tldDataset.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/tracking/test/test_tldDataset.cpp b/modules/tracking/test/test_tldDataset.cpp index 63cb378c1..abc50ab11 100644 --- a/modules/tracking/test/test_tldDataset.cpp +++ b/modules/tracking/test/test_tldDataset.cpp @@ -39,7 +39,6 @@ // //M*/ -#include #include #include From 2c9b8e8307f7c475dc3eee4f1a1b861a89ef4622 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 23:16:44 +0900 Subject: [PATCH 19/98] Fixed build error --- modules/tracking/src/tldDataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 2f276306a..c816858b0 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -129,7 +129,7 @@ namespace cv else if (frameNum < 1000) strcat(fullPath, "00"); else if (frameNum < 10000) strcat(fullPath, "0"); - _itoa(frameNum, numStr, 10); + sprintf(numStr, "%d", frameNum); strcat(fullPath, numStr); if (flagPNG) strcat(fullPath, ".png"); else strcat(fullPath, ".jpg"); From 3c200866d82a07673e056743dd946f604c0b9f52 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 23:32:07 +0900 Subject: [PATCH 20/98] Fix --- .../tracking/{test/test_tldDataset.cpp => samples/tld_test.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modules/tracking/{test/test_tldDataset.cpp => samples/tld_test.cpp} (100%) diff --git a/modules/tracking/test/test_tldDataset.cpp b/modules/tracking/samples/tld_test.cpp similarity index 100% rename from modules/tracking/test/test_tldDataset.cpp rename to modules/tracking/samples/tld_test.cpp From 0c610a5e63a2749081f398b4ced5d9534469a5d5 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 8 Jun 2015 23:45:52 +0900 Subject: [PATCH 21/98] Fix #2 --- modules/tracking/samples/tld_test.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/tracking/samples/tld_test.cpp b/modules/tracking/samples/tld_test.cpp index abc50ab11..41ece23b6 100644 --- a/modules/tracking/samples/tld_test.cpp +++ b/modules/tracking/samples/tld_test.cpp @@ -39,9 +39,10 @@ // //M*/ -#include -#include - +#include +#include +#include +#include #include using namespace std; From 59c0798cbd36c5690ef8fd3e1a69beb59778d5eb Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 00:03:34 +0900 Subject: [PATCH 22/98] Fix #3 --- modules/tracking/samples/tld_test.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/tracking/samples/tld_test.cpp b/modules/tracking/samples/tld_test.cpp index 41ece23b6..69d9b3970 100644 --- a/modules/tracking/samples/tld_test.cpp +++ b/modules/tracking/samples/tld_test.cpp @@ -93,7 +93,7 @@ static void onMouse(int event, int x, int y, int, void*) } } -void main() +int main(int argc, char** argv) { // // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" @@ -224,4 +224,6 @@ void main() waitKey(0); + + return 0; } \ No newline at end of file From 303c1c507eb3fd1d1125af175e5f65ebb114ef41 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 02:41:24 +0900 Subject: [PATCH 23/98] Fix #4 --- modules/tracking/samples/tld_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/samples/tld_test.cpp b/modules/tracking/samples/tld_test.cpp index 69d9b3970..ac89968a6 100644 --- a/modules/tracking/samples/tld_test.cpp +++ b/modules/tracking/samples/tld_test.cpp @@ -110,7 +110,7 @@ int main(int argc, char** argv) { cout << "***Error in the instantiation of the tracker...***\n"; getchar(); - return; + return 0; } //Get the first frame @@ -186,7 +186,7 @@ int main(int argc, char** argv) if (!tracker->init(frame, boundingBox)) { cout << "***Could not initialize tracker...***\n"; - return; + return 0; } initialized = true; rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); From 603fb880e5b506780a9c282eeaf6259fc075a612 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 11:31:10 +0900 Subject: [PATCH 24/98] Fixed Warnings #1 --- .../include/opencv2/tracking/tldDataset.hpp | 2 +- modules/tracking/samples/tld_test.cpp | 18 +++++++++--------- modules/tracking/src/tldDataset.cpp | 2 +- modules/tracking/src/tldModel.cpp | 8 ++++---- modules/tracking/src/tldTracker.cpp | 4 +--- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index 00d8dd711..6d21a7d48 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -48,7 +48,7 @@ namespace cv { namespace tld { - CV_EXPORTS cv::Rect2d tld_InitDataset(int datasetInd, char* rootPath = "TLD_dataset"); + CV_EXPORTS cv::Rect2d tld_InitDataset(int datasetInd, const char* rootPath = "TLD_dataset"); CV_EXPORTS cv::Mat tld_getNextDatasetFrame(); } } diff --git a/modules/tracking/samples/tld_test.cpp b/modules/tracking/samples/tld_test.cpp index ac89968a6..472d2c92d 100644 --- a/modules/tracking/samples/tld_test.cpp +++ b/modules/tracking/samples/tld_test.cpp @@ -93,7 +93,7 @@ static void onMouse(int event, int x, int y, int, void*) } } -int main(int argc, char** argv) +int main() { // // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" @@ -152,12 +152,12 @@ int main(int argc, char** argv) int frameCounter = 0; //Time measurment - float e3 = getTickCount(); + int64 e3 = getTickCount(); for (;;) { //Time measurment - float e1 = getTickCount(); + int64 e1 = getTickCount(); //Frame num frameCounter++; if (frameCounter == NUM_TEST_FRAMES) break; @@ -208,18 +208,18 @@ int main(int argc, char** argv) //Time measurment - float e2 = getTickCount(); - float t1 = (e2 - e1) / getTickFrequency(); - cout << frameCounter << "\tframe : " << t1 * 1000 << "ms" << endl; + int64 e2 = getTickCount(); + double t1 = (e2 - e1) / getTickFrequency(); + cout << frameCounter << "\tframe : " << t1 * 1000.0 << "ms" << endl; //waitKey(0); } } //Time measurment - float e4 = getTickCount(); - float t2 = (e4 - e3) / getTickFrequency(); - cout << "Average Time for Frame: " << t2 * 1000 / frameCounter << "ms" << endl; + int64 e4 = getTickCount(); + double t2 = (e4 - e3) / getTickFrequency(); + cout << "Average Time for Frame: " << t2 * 1000.0 / frameCounter << "ms" << endl; cout << "Average FPS: " << 1.0 / t2*frameCounter << endl; diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index c816858b0..81ecd6f72 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -49,7 +49,7 @@ namespace cv int frameNum = 0; bool flagPNG = false; - cv::Rect2d tld_InitDataset(int datasetInd, char* rootPath) + cv::Rect2d tld_InitDataset(int datasetInd,const char* rootPath) { char* folderName = ""; int x, y, w, h; diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 4b0e07100..15e7b945e 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -263,11 +263,11 @@ namespace cv (*proxyN)++; } - void TrackerTLDModel::printme(FILE* port) + void TrackerTLDModel::printme(FILE* port) { - //dfprintf((port, "TrackerTLDModel:\n")); - //dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); - //dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); + dfprintf((port, "TrackerTLDModel:\n")); + dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); + dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); } diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index c55f281b4..962a0f7c4 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -89,9 +89,7 @@ bool TrackerTLDImpl::initImpl(const Mat& image, const Rect2d& boundingBox) myBoundingBox.width *= scale; myBoundingBox.height *= scale; } - model = Ptr(new TrackerTLDModel(params, image_gray, myBoundingBox, data->getMinSize())); - TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast(model)); - + model = Ptr(new TrackerTLDModel(params, image_gray, myBoundingBox, data->getMinSize())); data->confident = false; data->failedLastTime = false; From 15da8085560ac820f9ed3e963d544c9381ca0610 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 14:40:10 +0900 Subject: [PATCH 25/98] Fixed Warnings #2 --- modules/tracking/src/tldDataset.cpp | 22 +++++++++++----------- modules/tracking/src/tldDetector.cpp | 6 +++--- modules/tracking/src/tldDetector.hpp | 2 +- modules/tracking/src/tldModel.cpp | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 81ecd6f72..8432fc272 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -51,61 +51,61 @@ namespace cv cv::Rect2d tld_InitDataset(int datasetInd,const char* rootPath) { - char* folderName = ""; + char* folderName = (char *)""; int x, y, w, h; flagPNG = false; frameNum = 1; if (datasetInd == 1) { - folderName = "01_david"; + folderName = (char *)"01_david"; x = 165, y = 83; w = 51; h = 54; frameNum = 100; } if (datasetInd == 2) { - folderName = "02_jumping"; + folderName = (char *)"02_jumping"; x = 147, y = 110; w = 33; h = 32; } if (datasetInd == 3) { - folderName = "03_pedestrian1"; + folderName = (char *)"03_pedestrian1"; x = 47, y = 51; w = 21; h = 36; } if (datasetInd == 4) { - folderName = "04_pedestrian2"; + folderName = (char *)"04_pedestrian2"; x = 130, y = 134; w = 21; h = 53; } if (datasetInd == 5) { - folderName = "05_pedestrian3"; + folderName = (char *)"05_pedestrian3"; x = 154, y = 102; w = 24; h = 52; } if (datasetInd == 6) { - folderName = "06_car"; + folderName = (char *)"06_car"; x = 142, y = 125; w = 90; h = 39; } if (datasetInd == 7) { - folderName = "07_motocross"; + folderName = (char *)"07_motocross"; x = 290, y = 43; w = 23; h = 40; flagPNG = true; } if (datasetInd == 8) { - folderName = "08_volkswagen"; + folderName = (char *)"08_volkswagen"; x = 273, y = 77; w = 27; h = 25; } if (datasetInd == 9) { - folderName = "09_carchase"; + folderName = (char *)"09_carchase"; x = 145, y = 84; w = 54; h = 37; } if (datasetInd == 10){ - folderName = "10_panda"; + folderName = (char *)"10_panda"; x = 58, y = 100; w = 27; h = 22; } diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 8a11ecdff..95f6eaf63 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -159,7 +159,7 @@ namespace cv { LabeledPatch labPatch; total++; - if (!patchVariance(intImgP, intImgP2, *originalVariance, Point(dx * i, dy * j), initSize)) + if (!patchVariance(intImgP, intImgP2, originalVariance, Point(dx * i, dy * j), initSize)) continue; if (ensembleClassifierNum(&blurred_img.at(dy * j, dx * i)) <= ENSEMBLE_THRESHOLD) continue; @@ -209,7 +209,7 @@ namespace cv // Computes the variance of subimage given by box, with the help of two integral // images intImgP and intImgP2 (sum of squares), which should be also provided. - bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size) + bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size) { int x = (pt.x), y = (pt.y), width = (size.width), height = (size.height); CV_Assert(0 <= x && (x + width) < intImgP.cols && (x + width) < intImgP2.cols); @@ -229,7 +229,7 @@ namespace cv D = intImgP2(y + height, x + width); p2 = (A + D - B - C) / (width * height); - return ((p2 - p * p) > VARIANCE_THRESHOLD * originalVariance); + return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariance); } } diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index b51108e23..6e8891d26 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -94,7 +94,7 @@ namespace cv friend class MyMouseCallbackDEBUG; void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } - inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double originalVariance, Point pt, Size size); + inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); }; } } diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 15e7b945e..f8fcaa3b2 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -46,8 +46,8 @@ namespace cv namespace tld { //Constructor - TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize) :minSize_(minSize), - timeStampPositiveNext(0), timeStampNegativeNext(0), params_(params), boundingBox_(boundingBox) + TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize): + timeStampPositiveNext(0), timeStampNegativeNext(0), minSize_(minSize), params_(params), boundingBox_(boundingBox) { std::vector closest, scanGrid; Mat scaledImg, blurredImg, image_blurred; From 237ca8172d4e26397ef720ee130d479f8084d599 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 14:54:41 +0900 Subject: [PATCH 26/98] Fixed Warnings #3 --- modules/tracking/src/tldDetector.cpp | 6 +++--- modules/tracking/src/tldDetector.hpp | 2 +- modules/tracking/src/tldModel.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 95f6eaf63..c6cbf563a 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -159,7 +159,7 @@ namespace cv { LabeledPatch labPatch; total++; - if (!patchVariance(intImgP, intImgP2, originalVariance, Point(dx * i, dy * j), initSize)) + if (!patchVariance(intImgP, intImgP2, originalVariancePtr, Point(dx * i, dy * j), initSize)) continue; if (ensembleClassifierNum(&blurred_img.at(dy * j, dx * i)) <= ENSEMBLE_THRESHOLD) continue; @@ -209,7 +209,7 @@ namespace cv // Computes the variance of subimage given by box, with the help of two integral // images intImgP and intImgP2 (sum of squares), which should be also provided. - bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size) + bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariancePtr, Point pt, Size size) { int x = (pt.x), y = (pt.y), width = (size.width), height = (size.height); CV_Assert(0 <= x && (x + width) < intImgP.cols && (x + width) < intImgP2.cols); @@ -229,7 +229,7 @@ namespace cv D = intImgP2(y + height, x + width); p2 = (A + D - B - C) / (width * height); - return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariance); + return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariancePtr); } } diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 6e8891d26..f675fd2a6 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -79,7 +79,7 @@ namespace cv std::vector classifiers; std::vector > *positiveExamples, *negativeExamples; std::vector *timeStampsPositive, *timeStampsNegative; - double *originalVariance; + double *originalVariancePtr; static void generateScanGrid(int rows, int cols, Size initBox, std::vector& res, bool withScaling = false); struct LabeledPatch diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index f8fcaa3b2..2bc10a76b 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -60,7 +60,7 @@ namespace cv detector->negativeExamples = &negativeExamples; detector->timeStampsPositive = &timeStampsPositive; detector->timeStampsNegative = &timeStampsNegative; - detector->originalVariance = &originalVariance_; + detector->originalVariancePtr = &originalVariance_; //Calculate the variance in initial BB originalVariance_ = variance(image(boundingBox)); From 1c70b5a014f9454e485bb8c2d7d87411ac98f385 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 9 Jun 2015 15:11:41 +0900 Subject: [PATCH 27/98] Shadow Fix --- modules/tracking/src/tldDetector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index c6cbf563a..714d1330e 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -209,7 +209,7 @@ namespace cv // Computes the variance of subimage given by box, with the help of two integral // images intImgP and intImgP2 (sum of squares), which should be also provided. - bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariancePtr, Point pt, Size size) + bool TLDDetector::patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size) { int x = (pt.x), y = (pt.y), width = (size.width), height = (size.height); CV_Assert(0 <= x && (x + width) < intImgP.cols && (x + width) < intImgP2.cols); @@ -229,7 +229,7 @@ namespace cv D = intImgP2(y + height, x + width); p2 = (A + D - B - C) / (width * height); - return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariancePtr); + return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariance); } } From 7b221052843791acf0b8564ff6e1421cd95ae1e8 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 10 Jun 2015 05:07:07 +0900 Subject: [PATCH 28/98] Split the training coefficient into numerator and denumerator --- .../include/opencv2/tracking/tracker.hpp | 2 + modules/tracking/src/trackerKCF.cpp | 66 +++++++++++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index e43dbd45b..1b368c4e0 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1210,6 +1210,8 @@ class CV_EXPORTS_W TrackerKCF : public Tracker double interp_factor; // linear interpolation factor for adaptation double output_sigma_factor; // spatial bandwidth (proportional to target) bool resize; // activate the resize feature to improve the processing speed + bool splitCoeff; // split the training coefficients into two matrices + bool wrapKernel; // wrap around the kernel values int max_patch_size; // threshold for the ROI size MODE descriptor; // descriptor type }; diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 73d645787..845be7849 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -97,6 +97,7 @@ namespace cv{ void extractCN(Mat _patch, Mat & cnFeatures) const; void denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k) const; void calcResponse(const Mat _alphaf, const Mat _k, Mat & _response) const; + void calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response)const; void shiftRows(Mat& mat) const; void shiftRows(Mat& mat, int n) const; @@ -110,8 +111,10 @@ namespace cv{ Mat y,yf; // training response and its FFT Mat x,xf; // observation and its FFT Mat k,kf; // dense gaussian kernel and its FFT - Mat new_alphaf, alphaf; // learning rate - Mat z, new_z; + Mat kf_lambda; // kf+lambda + Mat new_alphaf, alphaf; // training coefficients + Mat new_alphaf_den, alphaf_den; // for splitted training coefficients + Mat z, new_z; // model Mat response; // detection result bool resizeImage; // resize the image whenever needed and the patch size is large @@ -219,7 +222,11 @@ namespace cv{ // detection part if(frame>0){ denseGaussKernel(params.sigma,x,z,k); - calcResponse(alphaf,k,response); + if(params.splitCoeff){ + calcResponse(alphaf,alphaf_den,k,response); + }else{ + calcResponse(alphaf,k,response); + } minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); roi.x+=(maxLoc.x-roi.width/2+1);roi.y+=(maxLoc.y-roi.height/2+1); @@ -235,7 +242,7 @@ namespace cv{ denseGaussKernel(params.sigma,x,x,k); fft2(k,kf); - kf=kf+params.lambda; + kf_lambda=kf+params.lambda; /* TODO: optimize this element-wise division * new_alphaf=yf./kf @@ -243,11 +250,17 @@ namespace cv{ */ new_alphaf=Mat_(yf.rows, yf.cols); std::complex temp; - for(int i=0;i(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf.at(i,j)[0],kf.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); - new_alphaf.at(i,j)[0]=temp.real(); - new_alphaf.at(i,j)[1]=temp.imag(); + + if(params.splitCoeff){ + mulSpectrums(yf,kf,new_alphaf,0); + mulSpectrums(kf,kf_lambda,new_alphaf_den,0); + }else{ + for(int i=0;i(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf_lambda.at(i,j)[0],kf_lambda.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); + new_alphaf.at(i,j)[0]=temp.real(); + new_alphaf.at(i,j)[1]=temp.imag(); + } } } @@ -255,9 +268,11 @@ namespace cv{ new_z=x.clone(); if(frame==0){ alphaf=new_alphaf.clone(); + if(params.splitCoeff)alphaf_den=new_alphaf_den.clone(); z=x.clone(); }else{ alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; + if(params.splitCoeff)alphaf_den=(1.0-params.interp_factor)*alphaf_den+params.interp_factor*new_alphaf_den; z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; } @@ -468,9 +483,12 @@ namespace cv{ pixelWiseMult(_xf,_yf,xyf_v,0,true); sumChannels(xyf_v,xyf); ifft2(xyf,xyf); -// shiftRows(xyf, _x.rows/2); -// shiftCols(xyf, _x.cols/2); - + + if(params.wrapKernel){ + shiftRows(xyf, _x.rows/2); + shiftCols(xyf, _x.cols/2); + } + //(xx + yy - 2 * xy) / numel(x) xy=(normX+normY-2*xyf)/(_x.rows*_x.cols*_x.channels()); @@ -559,6 +577,26 @@ namespace cv{ mulSpectrums(_alphaf,_kf,spec,0,false); ifft2(spec,_response); } + + void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response)const { + Mat _kf; + fft2(_k,_kf); + Mat spec; + Mat spec2=Mat_(_k.rows, _k.cols); + std::complex temp; + + mulSpectrums(_alphaf,_kf,spec,0,false); + + for(int i=0;i<_k.rows;i++){ + for(int j=0;j<_k.cols;j++){ + temp=std::complex(spec.at(i,j)[0],spec.at(i,j)[1])/(std::complex(_alphaf_den.at(i,j)[0],_alphaf_den.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); + spec2.at(i,j)[0]=temp.real(); + spec2.at(i,j)[1]=temp.imag(); + } + } + + ifft2(spec2,_response); + } /*----------------------------------------------------------------------*/ /* @@ -570,8 +608,10 @@ namespace cv{ interp_factor=0.075; output_sigma_factor=1.0/16.0; resize=true; - max_patch_size=100*100; + max_patch_size=80*80; descriptor=CN; + splitCoeff=true; + wrapKernel=false; } void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){} From ab3015d4b0b9631aa8bac2630d49c44f8ada15b2 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Thu, 18 Jun 2015 04:49:29 +0900 Subject: [PATCH 29/98] Added the feature compression method --- .../include/opencv2/tracking/tracker.hpp | 5 +- modules/tracking/src/trackerKCF.cpp | 97 +++++++++++++++++-- 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 1b368c4e0..8de034d16 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1208,11 +1208,14 @@ class CV_EXPORTS_W TrackerKCF : public Tracker double sigma; // gaussian kernel bandwidth double lambda; // regularization double interp_factor; // linear interpolation factor for adaptation - double output_sigma_factor; // spatial bandwidth (proportional to target) + double output_sigma_factor; // spatial bandwidth (proportional to target) + double pca_learning_rate; // compression learning rate bool resize; // activate the resize feature to improve the processing speed bool splitCoeff; // split the training coefficients into two matrices bool wrapKernel; // wrap around the kernel values + bool compressFeature; // activate pca method to compress the features int max_patch_size; // threshold for the ROI size + int compressed_size; // feature size after compression MODE descriptor; // descriptor type }; diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 845be7849..10df5af9e 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -93,11 +93,13 @@ namespace cv{ void inline ifft2(const Mat src, Mat & dest) const ; void inline pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB=false) const; void inline sumChannels(std::vector src, Mat & dest) const; + void inline updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & _proj_mtx,double pca_rate, int compressed_sz) const; + void inline compress(const Mat _proj_mtx, const Mat src, Mat & dest) const; bool getSubWindow(const Mat img, const Rect roi, Mat& patch) const; void extractCN(Mat _patch, Mat & cnFeatures) const; void denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k) const; void calcResponse(const Mat _alphaf, const Mat _k, Mat & _response) const; - void calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response)const; + void calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response) const; void shiftRows(Mat& mat) const; void shiftRows(Mat& mat, int n) const; @@ -116,6 +118,7 @@ namespace cv{ Mat new_alphaf_den, alphaf_den; // for splitted training coefficients Mat z, new_z; // model Mat response; // detection result + Mat old_cov_mtx, proj_mtx; // for feature compression bool resizeImage; // resize the image whenever needed and the patch size is large @@ -208,6 +211,7 @@ namespace cv{ bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){ double minVal, maxVal; // min-max response Point minLoc,maxLoc; // min-max location + Mat zc; Mat img=image.clone(); // check the channels of the input image, grayscale is preferred @@ -221,14 +225,26 @@ namespace cv{ // detection part if(frame>0){ - denseGaussKernel(params.sigma,x,z,k); + //compute the gaussian kernel + if(params.compressFeature){ + compress(proj_mtx,x,x); + compress(proj_mtx,z,zc); + denseGaussKernel(params.sigma,x,zc,k); + }else{ + denseGaussKernel(params.sigma,x,z,k); + } + + // calculate filter response if(params.splitCoeff){ calcResponse(alphaf,alphaf_den,k,response); }else{ calcResponse(alphaf,k,response); } + + // extract the maximum response minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); - roi.x+=(maxLoc.x-roi.width/2+1);roi.y+=(maxLoc.y-roi.height/2+1); + roi.x+=(maxLoc.x-roi.width/2+1); + roi.y+=(maxLoc.y-roi.height/2+1); // update the bounding box boundingBox.x=(resizeImage?roi.x*2:roi.x)+boundingBox.width/2; @@ -238,6 +254,20 @@ namespace cv{ // extract the patch for learning purpose if(!getSubWindow(img,roi, x))return false; + //update the training data + new_z=x.clone(); + if(frame==0){ + z=x.clone(); + }else{ + z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; + } + + if(params.compressFeature){ + // feature compression + updateProjectionMatrix(z,old_cov_mtx,proj_mtx,params.pca_learning_rate,params.compressed_size); + compress(proj_mtx,x,x); + } + // Kernel Regularized Least-Squares, calculate alphas denseGaussKernel(params.sigma,x,x,k); @@ -264,16 +294,13 @@ namespace cv{ } } - // update the learning model - new_z=x.clone(); + // update the RLS model if(frame==0){ alphaf=new_alphaf.clone(); if(params.splitCoeff)alphaf_den=new_alphaf_den.clone(); - z=x.clone(); }else{ alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; if(params.splitCoeff)alphaf_den=(1.0-params.interp_factor)*alphaf_den+params.interp_factor*new_alphaf_den; - z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; } frame++; @@ -384,6 +411,54 @@ namespace cv{ dest+=src[i]; } } + + /* + * obtains the projection matrix using PCA + */ + void inline TrackerKCFImpl::updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & _proj_mtx, double pca_rate, int compressed_sz)const{ + CV_Assert(compressed_sz<=src.channels()); + + // compute average + std::vector layers(src.channels()); + std::vector average(src.channels()); + split(src,layers); + + for (int i=0;i(i,i)=w.at(i); + } + + // update the covariance matrix + old_cov=(1.0-pca_rate)*old_cov+pca_rate*_proj_mtx*proj_vars*_proj_mtx.t(); + } + + /* + * compress the features + */ + void inline TrackerKCFImpl::compress(const Mat _proj_mtx, const Mat src, Mat & dest)const{ + Mat data=src.reshape(1,src.rows*src.cols); + Mat compressed=data*_proj_mtx; + dest=compressed.reshape(_proj_mtx.cols,src.rows).clone(); + } /* * obtain the patch and apply hann window filter to it @@ -578,6 +653,9 @@ namespace cv{ ifft2(spec,_response); } + /* + * calculate the detection response for splitted form + */ void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response)const { Mat _kf; fft2(_k,_kf); @@ -612,6 +690,11 @@ namespace cv{ descriptor=CN; splitCoeff=true; wrapKernel=false; + + //feature compression + compressFeature=true; + compressed_size=2; + pca_learning_rate=0.15; } void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){} From 55db204590854b394c27c7782856a99f99ee7c35 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Thu, 18 Jun 2015 05:08:11 +0900 Subject: [PATCH 30/98] Fixing some indentations --- modules/tracking/src/trackerKCF.cpp | 47 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 10df5af9e..1536d7ac1 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -225,30 +225,28 @@ namespace cv{ // detection part if(frame>0){ - //compute the gaussian kernel - if(params.compressFeature){ - compress(proj_mtx,x,x); - compress(proj_mtx,z,zc); - denseGaussKernel(params.sigma,x,zc,k); - }else{ - denseGaussKernel(params.sigma,x,z,k); - } - - // calculate filter response - if(params.splitCoeff){ + //compute the gaussian kernel + if(params.compressFeature){ + compress(proj_mtx,x,x); + compress(proj_mtx,z,zc); + denseGaussKernel(params.sigma,x,zc,k); + }else + denseGaussKernel(params.sigma,x,z,k); + + // calculate filter response + if(params.splitCoeff) calcResponse(alphaf,alphaf_den,k,response); - }else{ + else calcResponse(alphaf,k,response); - } - - // extract the maximum response - minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); - roi.x+=(maxLoc.x-roi.width/2+1); - roi.y+=(maxLoc.y-roi.height/2+1); - - // update the bounding box - boundingBox.x=(resizeImage?roi.x*2:roi.x)+boundingBox.width/2; - boundingBox.y=(resizeImage?roi.y*2:roi.y)+boundingBox.height/2; + + // extract the maximum response + minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); + roi.x+=(maxLoc.x-roi.width/2+1); + roi.y+=(maxLoc.y-roi.height/2+1); + + // update the bounding box + boundingBox.x=(resizeImage?roi.x*2:roi.x)+boundingBox.width/2; + boundingBox.y=(resizeImage?roi.y*2:roi.y)+boundingBox.height/2; } // extract the patch for learning purpose @@ -256,11 +254,10 @@ namespace cv{ //update the training data new_z=x.clone(); - if(frame==0){ + if(frame==0) z=x.clone(); - }else{ + else z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; - } if(params.compressFeature){ // feature compression From 6280a069f21941cde7a0e7fda135fadbe1a4b03d Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Thu, 18 Jun 2015 05:11:17 +0900 Subject: [PATCH 31/98] Fixing some indentations --- modules/tracking/src/trackerKCF.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 1536d7ac1..f2887d03b 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -227,17 +227,17 @@ namespace cv{ if(frame>0){ //compute the gaussian kernel if(params.compressFeature){ - compress(proj_mtx,x,x); - compress(proj_mtx,z,zc); - denseGaussKernel(params.sigma,x,zc,k); + compress(proj_mtx,x,x); + compress(proj_mtx,z,zc); + denseGaussKernel(params.sigma,x,zc,k); }else - denseGaussKernel(params.sigma,x,z,k); + denseGaussKernel(params.sigma,x,z,k); // calculate filter response if(params.splitCoeff) - calcResponse(alphaf,alphaf_den,k,response); + calcResponse(alphaf,alphaf_den,k,response); else - calcResponse(alphaf,k,response); + calcResponse(alphaf,k,response); // extract the maximum response minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); From 244b7926beddb710b253f9b7dcfe4b45c2e40c3f Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Sat, 20 Jun 2015 04:57:20 +0900 Subject: [PATCH 32/98] Fixing alignments --- modules/tracking/src/trackerKCF.cpp | 107 +++++++++++++--------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index f2887d03b..0765ffb12 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -144,7 +144,7 @@ namespace cv{ params.read( fn ); } - void TrackerKCFImpl::write( cv::FileStorage& fs ) const{ + void TrackerKCFImpl::write( cv::FileStorage& fs ) const { params.write( fs ); } @@ -189,7 +189,7 @@ namespace cv{ y=Mat::zeros((int)roi.height,(int)roi.width,CV_64F); for(unsigned i=0;i(i,j)=(i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1); + y.at(i,j)=(i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1); } } @@ -283,11 +283,11 @@ namespace cv{ mulSpectrums(kf,kf_lambda,new_alphaf_den,0); }else{ for(int i=0;i(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf_lambda.at(i,j)[0],kf_lambda.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); - new_alphaf.at(i,j)[0]=temp.real(); - new_alphaf.at(i,j)[1]=temp.imag(); - } + for(int j=0;j(yf.at(i,j)[0],yf.at(i,j)[1])/(std::complex(kf_lambda.at(i,j)[0],kf_lambda.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); + new_alphaf.at(i,j)[0]=temp.real(); + new_alphaf.at(i,j)[1]=temp.imag(); + } } } @@ -312,7 +312,7 @@ namespace cv{ /* * hann window filter */ - void TrackerKCFImpl::createHanningWindow(OutputArray _dst, const cv::Size winSize, const int type)const{ + void TrackerKCFImpl::createHanningWindow(OutputArray _dst, const cv::Size winSize, const int type) const { CV_Assert( type == CV_32FC1 || type == CV_64FC1 ); _dst.create(winSize, type); @@ -325,27 +325,22 @@ namespace cv{ double coeff0 = 2.0 * CV_PI / (double)(cols - 1), coeff1 = 2.0f * CV_PI / (double)(rows - 1); for(int j = 0; j < cols; j++) - wc[j] = 0.5 * (1.0 - cos(coeff0 * j)); - - if(dst.depth() == CV_32F) - { - for(int i = 0; i < rows; i++) - { - float* dstData = dst.ptr(i); - double wr = 0.5 * (1.0 - cos(coeff1 * i)); - for(int j = 0; j < cols; j++) - dstData[j] = (float)(wr * wc[j]); - } - } - else - { - for(int i = 0; i < rows; i++) - { - double* dstData = dst.ptr(i); - double wr = 0.5 * (1.0 - cos(coeff1 * i)); - for(int j = 0; j < cols; j++) - dstData[j] = wr * wc[j]; - } + wc[j] = 0.5 * (1.0 - cos(coeff0 * j)); + + if(dst.depth() == CV_32F){ + for(int i = 0; i < rows; i++){ + float* dstData = dst.ptr(i); + double wr = 0.5 * (1.0 - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = (float)(wr * wc[j]); + } + }else{ + for(int i = 0; i < rows; i++){ + double* dstData = dst.ptr(i); + double wr = 0.5 * (1.0 - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = wr * wc[j]; + } } // perform batch sqrt for SSE performance gains @@ -355,7 +350,7 @@ namespace cv{ /* * simplification of fourier transform function in opencv */ - void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest)const { + void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest) const { std::vector layers(src.channels()); std::vector outputs(src.channels()); @@ -368,7 +363,7 @@ namespace cv{ merge(outputs,dest); } - void inline TrackerKCFImpl::fft2(const Mat src, std::vector & dest) const{ + void inline TrackerKCFImpl::fft2(const Mat src, std::vector & dest) const { std::vector layers(src.channels()); dest.clear(); dest.resize(src.channels()); @@ -383,14 +378,14 @@ namespace cv{ /* * simplification of inverse fourier transform function in opencv */ - void inline TrackerKCFImpl::ifft2(const Mat src, Mat & dest)const { + void inline TrackerKCFImpl::ifft2(const Mat src, Mat & dest) const { idft(src,dest,DFT_SCALE+DFT_REAL_OUTPUT); } /* * Point-wise multiplication of two Multichannel Mat data */ - void inline TrackerKCFImpl::pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const{ + void inline TrackerKCFImpl::pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const { dest.clear(); dest.resize(src1.size()); @@ -402,7 +397,7 @@ namespace cv{ /* * Combines all channels in a multi-channels Mat data into a single channel */ - void inline TrackerKCFImpl::sumChannels(std::vector src, Mat & dest) const{ + void inline TrackerKCFImpl::sumChannels(std::vector src, Mat & dest) const { dest=src[0].clone(); for(unsigned i=1;i1)cvtColor(patch,patch, CV_BGR2GRAY); - patch.convertTo(patch,CV_64F); - patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 - break; + if(img.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); + patch.convertTo(patch,CV_64F); + patch=patch/255.0-0.5; // normalize to range -0.5 .. 0.5 + break; case CN: - CV_Assert(img.channels() == 3); - extractCN(patch,patch); - break; + CV_Assert(img.channels() == 3); + extractCN(patch,patch); + break; case CN2: - if(patch.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); - break; + if(patch.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); + break; } patch=patch.mul(hann); // hann window filter @@ -523,13 +518,13 @@ namespace cv{ for(int i=0;i<_patch.rows;i++){ for(int j=0;j<_patch.cols;j++){ - pixel=_patch.at(i,j); - index=(unsigned)(floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8)); - - //copy the values - for(int _k=0;_k<10;_k++){ - temp.at >(i,j)[_k]=ColorNames[index][_k]; - } + pixel=_patch.at(i,j); + index=(unsigned)(floor(pixel[2]/8)+32*floor(pixel[1]/8)+32*32*floor(pixel[0]/8)); + + //copy the values + for(int _k=0;_k<10;_k++){ + temp.at >(i,j)[_k]=ColorNames[index][_k]; + } } } @@ -539,7 +534,7 @@ namespace cv{ /* * dense gauss kernel function */ - void TrackerKCFImpl::denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k)const{ + void TrackerKCFImpl::denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k) const { std::vector _xf,_yf,xyf_v; Mat xy,xyf; double normX, normY; @@ -641,7 +636,7 @@ namespace cv{ /* * calculate the detection response */ - void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _k, Mat & _response)const { + void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _k, Mat & _response) const { //alpha f--> 2channels ; k --> 1 channel; Mat _kf; fft2(_k,_kf); @@ -653,7 +648,7 @@ namespace cv{ /* * calculate the detection response for splitted form */ - void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response)const { + void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response) const { Mat _kf; fft2(_k,_kf); Mat spec; From a4e84573642c1aa2859e4c3ad0ee7398010b6482 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Sat, 20 Jun 2015 06:04:36 +0900 Subject: [PATCH 33/98] Fixing some alignments --- modules/tracking/src/trackerKCF.cpp | 61 ++++++++++++----------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 0765ffb12..9fca2880b 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -563,7 +563,7 @@ namespace cv{ //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) for(int i=0;i(i,j)<0.0)xy.at(i,j)=0.0; + if(xy.at(i,j)<0.0)xy.at(i,j)=0.0; } } @@ -584,8 +584,8 @@ namespace cv{ int _k = (mat.rows-1); mat.row(_k).copyTo(temp); for(; _k > 0 ; _k-- ) { - m = mat.row(_k); - mat.row(_k-1).copyTo(m); + m = mat.row(_k); + mat.row(_k-1).copyTo(m); } m = mat.row(0); temp.copyTo(m); @@ -594,42 +594,33 @@ namespace cv{ // circular shift n rows from up to down if n > 0, -n rows from down to up if n < 0 void TrackerKCFImpl::shiftRows(Mat& mat, int n) const { - if( n < 0 ) { - - n = -n; - flip(mat,mat,0); - for(int _k=0; _k < n;_k++) { - shiftRows(mat); - } - flip(mat,mat,0); - - } else { - - for(int _k=0; _k < n;_k++) { - shiftRows(mat); - } + n = -n; + flip(mat,mat,0); + for(int _k=0; _k < n;_k++) { + shiftRows(mat); + } + flip(mat,mat,0); + }else{ + for(int _k=0; _k < n;_k++) { + shiftRows(mat); + } } - } //circular shift n columns from left to right if n > 0, -n columns from right to left if n < 0 void TrackerKCFImpl::shiftCols(Mat& mat, int n) const { - if(n < 0){ - - n = -n; - flip(mat,mat,1); - transpose(mat,mat); - shiftRows(mat,n); - transpose(mat,mat); - flip(mat,mat,1); - - } else { - - transpose(mat,mat); - shiftRows(mat,n); - transpose(mat,mat); + n = -n; + flip(mat,mat,1); + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); + flip(mat,mat,1); + }else{ + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); } } @@ -659,9 +650,9 @@ namespace cv{ for(int i=0;i<_k.rows;i++){ for(int j=0;j<_k.cols;j++){ - temp=std::complex(spec.at(i,j)[0],spec.at(i,j)[1])/(std::complex(_alphaf_den.at(i,j)[0],_alphaf_den.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); - spec2.at(i,j)[0]=temp.real(); - spec2.at(i,j)[1]=temp.imag(); + temp=std::complex(spec.at(i,j)[0],spec.at(i,j)[1])/(std::complex(_alphaf_den.at(i,j)[0],_alphaf_den.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); + spec2.at(i,j)[0]=temp.real(); + spec2.at(i,j)[1]=temp.imag(); } } From 8719073735e4716bac86f8a80439f604b2413ee5 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 22 Jun 2015 21:14:43 +0900 Subject: [PATCH 34/98] Added OCL versions of Sr and Sc functions --- modules/tracking/src/opencl/tldDetector.cl | 69 +++++++ modules/tracking/src/precomp.hpp | 1 + modules/tracking/src/tldDetector.cpp | 229 ++++++++++++++++++++- modules/tracking/src/tldDetector.hpp | 6 + modules/tracking/src/tldModel.cpp | 30 ++- modules/tracking/src/tldModel.hpp | 2 + 6 files changed, 334 insertions(+), 3 deletions(-) create mode 100644 modules/tracking/src/opencl/tldDetector.cl diff --git a/modules/tracking/src/opencl/tldDetector.cl b/modules/tracking/src/opencl/tldDetector.cl new file mode 100644 index 000000000..9610bd72c --- /dev/null +++ b/modules/tracking/src/opencl/tldDetector.cl @@ -0,0 +1,69 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Copyright (C) 2014, Advanced Micro Devices, Inc., all rights reserved. +// Third party copyrights are property of their respective owners. + + + +__kernel void NCC(__global const uchar *patch, + __global const uchar *positiveSamples, + __global const uchar *negativeSamples, + __global float *ncc, + int posNum, + int negNum) +{ + int id = get_global_id(0); + if (id >= 1000) return; + bool posFlg; + + if (id < 500) + posFlg = true; + if (id >= 500) + { + //Negative index + id = id - 500; + posFlg = false; + } + + //Variables + int s1 = 0, s2 = 0, n1 = 0, n2 = 0, prod = 0; + float sq1 = 0, sq2 = 0, ares = 0; + int N = 225; + //NCC with positive patch + if (posFlg && id < posNum) + { + for (int i = 0; i < N; i++) + { + + s1 += positiveSamples[id * N + i]; + s2 += patch[i]; + n1 += positiveSamples[id * N + i] * positiveSamples[id * N + i]; + n2 += patch[i] * patch[i]; + prod += positiveSamples[id * N + i] * patch[i]; + } + sq1 = sqrt(max(0.0, n1 - 1.0 * s1 * s1 / N)); + sq2 = sqrt(max(0.0, n2 - 1.0 * s2 * s2 / N)); + ares = (sq2 == 0) ? sq1 / fabs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; + ncc[id] = ares; + } + + //NCC with negative patch + if (!posFlg && id < negNum) + { + for (int i = 0; i < N; i++) + { + + s1 += negativeSamples[id * N + i]; + s2 += patch[i]; + n1 += negativeSamples[id * N + i] * negativeSamples[id * N + i]; + n2 += patch[i] * patch[i]; + prod += negativeSamples[id * N + i] * patch[i]; + } + sq1 = sqrt(max(0.0, n1 - 1.0 * s1 * s1 / N)); + sq2 = sqrt(max(0.0, n2 - 1.0 * s2 * s2 / N)); + ares = (sq2 == 0) ? sq1 / fabs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; + ncc[id+500] = ares; + } +} diff --git a/modules/tracking/src/precomp.hpp b/modules/tracking/src/precomp.hpp index 98fbce135..c2c415835 100644 --- a/modules/tracking/src/precomp.hpp +++ b/modules/tracking/src/precomp.hpp @@ -44,5 +44,6 @@ #include "opencv2/tracking.hpp" #include "opencv2/core/utility.hpp" +#include "opencv2/core/ocl.hpp" #endif diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 714d1330e..b98f9356d 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -65,11 +65,119 @@ namespace cv // Calculate Relative similarity of the patch (NN-Model) double TLDDetector::Sr(const Mat_& patch) { + /* + int64 e1, e2; + float t; + e1 = getTickCount(); double splus = 0.0, sminus = 0.0; for (int i = 0; i < (int)(*positiveExamples).size(); i++) splus = std::max(splus, 0.5 * (NCC((*positiveExamples)[i], patch) + 1.0)); for (int i = 0; i < (int)(*negativeExamples).size(); i++) sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + printf("Sr: %f\n", t); + if (splus + sminus == 0.0) + return 0.0; + return splus / (sminus + splus); + */ + int64 e1, e2; + float t; + e1 = getTickCount(); + double splus = 0.0, sminus = 0.0; + Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + for (int i = 0; i < *posNum; i++) + { + modelSample.data = &(posExp->data[i * 225]); + splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0)); + } + for (int i = 0; i < *negNum; i++) + { + modelSample.data = &(negExp->data[i * 225]); + sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); + } + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + printf("Sr CPU: %f\n", t); + if (splus + sminus == 0.0) + return 0.0; + return splus / (sminus + splus); + } + + double TLDDetector::ocl_Sr(const Mat_& patch) + { + int64 e1, e2, e3, e4; + float t; + e1 = getTickCount(); + double splus = 0.0, sminus = 0.0; + + e3 = getTickCount(); + + UMat devPatch = patch.getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devPositiveSamples = posExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devNegativeSamples = negExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devNCC(1, 2*MAX_EXAMPLES_IN_MODEL, CV_32FC1, ACCESS_RW, USAGE_ALLOCATE_DEVICE_MEMORY); + + + ocl::Kernel k; + ocl::ProgramSource src = ocl::tracking::tldDetector_oclsrc; + String error; + ocl::Program prog(src, NULL, error); + k.create("NCC", prog); + if (k.empty()) + printf("Kernel create failed!!!\n"); + k.args( + ocl::KernelArg::PtrReadOnly(devPatch), + ocl::KernelArg::PtrReadOnly(devPositiveSamples), + ocl::KernelArg::PtrReadOnly(devNegativeSamples), + ocl::KernelArg::PtrWriteOnly(devNCC), + (int)posNum, + (int)negNum); + + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Mem Cpy GPU: %f\n", t); + + size_t globSize = 1000; + size_t localSize = 128; + e3 = getTickCount(); + if (!k.run(1, &globSize, &localSize, true)) + printf("Kernel Run Error!!!"); + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Kernel Run GPU: %f\n", t); + + e3 = getTickCount(); + Mat resNCC = devNCC.getMat(ACCESS_READ); + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Read Mem GPU: %f\n", t); + + ////Compare + //Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + //for (int i = 0; i < 200; i+=17) + //{ + // modelSample.data = &(posExp->data[i * 225]); + // printf("%f\t%f\n\n", resNCC.at(i), NCC(modelSample, patch)); + //} + + //for (int i = 0; i < 200; i+=23) + //{ + // modelSample.data = &(negExp->data[i * 225]); + // printf("%f\t%f\n", resNCC.at(500+i), NCC(modelSample, patch)); + //} + + + for (int i = 0; i < *posNum; i++) + splus = std::max(splus, 0.5 * (resNCC.at(i) + 1.0)); + + for (int i = 0; i < *negNum; i++) + sminus = std::max(sminus, 0.5 * (resNCC.at(i+500) +1.0)); + + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Sr GPU: %f\n\n", t); + if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); @@ -78,6 +186,10 @@ namespace cv // Calculate Conservative similarity of the patch (NN-Model) double TLDDetector::Sc(const Mat_& patch) { + /* + int64 e1, e2; + float t; + e1 = getTickCount(); double splus = 0.0, sminus = 0.0; int med = getMedian((*timeStampsPositive)); for (int i = 0; i < (int)(*positiveExamples).size(); i++) @@ -87,6 +199,118 @@ namespace cv } for (int i = 0; i < (int)(*negativeExamples).size(); i++) sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + printf("Sc: %f\n", t); + if (splus + sminus == 0.0) + return 0.0; + + return splus / (sminus + splus); + */ + + int64 e1, e2; + float t; + e1 = getTickCount(); + double splus = 0.0, sminus = 0.0; + Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + int med = getMedian((*timeStampsPositive)); + for (int i = 0; i < *posNum; i++) + { + if ((int)(*timeStampsPositive)[i] <= med) + { + modelSample.data = &(posExp->data[i * 225]); + splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0)); + } + } + for (int i = 0; i < *negNum; i++) + { + modelSample.data = &(negExp->data[i * 225]); + sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); + } + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + printf("Sc: %f\n", t); + if (splus + sminus == 0.0) + return 0.0; + + return splus / (sminus + splus); + } + + double TLDDetector::ocl_Sc(const Mat_& patch) + { + int64 e1, e2, e3, e4; + float t; + e1 = getTickCount(); + double splus = 0.0, sminus = 0.0; + + e3 = getTickCount(); + + UMat devPatch = patch.getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devPositiveSamples = posExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devNegativeSamples = negExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); + UMat devNCC(1, 2 * MAX_EXAMPLES_IN_MODEL, CV_32FC1, ACCESS_RW, USAGE_ALLOCATE_DEVICE_MEMORY); + + + ocl::Kernel k; + ocl::ProgramSource src = ocl::tracking::tldDetector_oclsrc; + String error; + ocl::Program prog(src, NULL, error); + k.create("NCC", prog); + if (k.empty()) + printf("Kernel create failed!!!\n"); + k.args( + ocl::KernelArg::PtrReadOnly(devPatch), + ocl::KernelArg::PtrReadOnly(devPositiveSamples), + ocl::KernelArg::PtrReadOnly(devNegativeSamples), + ocl::KernelArg::PtrWriteOnly(devNCC), + (int)posNum, + (int)negNum); + + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Mem Cpy GPU: %f\n", t); + + size_t globSize = 1000; + size_t localSize = 128; + e3 = getTickCount(); + if (!k.run(1, &globSize, &localSize, true)) + printf("Kernel Run Error!!!"); + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Kernel Run GPU: %f\n", t); + + e3 = getTickCount(); + Mat resNCC = devNCC.getMat(ACCESS_READ); + e4 = getTickCount(); + t = (e4 - e3) / getTickFrequency()*1000.0; + //printf("Read Mem GPU: %f\n", t); + + ////Compare + //Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); + //for (int i = 0; i < 200; i+=17) + //{ + // modelSample.data = &(posExp->data[i * 225]); + // printf("%f\t%f\n\n", resNCC.at(i), NCC(modelSample, patch)); + //} + + //for (int i = 0; i < 200; i+=23) + //{ + // modelSample.data = &(negExp->data[i * 225]); + // printf("%f\t%f\n", resNCC.at(500+i), NCC(modelSample, patch)); + //} + + int med = getMedian((*timeStampsPositive)); + for (int i = 0; i < *posNum; i++) + if ((int)(*timeStampsPositive)[i] <= med) + splus = std::max(splus, 0.5 * (resNCC.at(i) +1.0)); + + for (int i = 0; i < *negNum; i++) + sminus = std::max(sminus, 0.5 * (resNCC.at(i + 500) + 1.0)); + + e2 = getTickCount(); + t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Sc GPU: %f\n\n", t); + if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); @@ -167,7 +391,8 @@ namespace cv labPatch.rect = Rect2d(dx * i * scale, dy * j * scale, initSize.width * scale, initSize.height * scale); resample(resized_img, Rect2d(Point(dx * i, dy * j), initSize), standardPatch); - tmp = Sr(standardPatch); + + tmp = ocl_Sr(standardPatch); ////To fix: Check the paper, probably this cause wrong learning // @@ -185,7 +410,7 @@ namespace cv { npos++; } - tmp = Sc(standardPatch); + tmp = ocl_Sc(standardPatch); if (tmp > maxSc) { maxSc = tmp; diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index f675fd2a6..f86e7e804 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -43,6 +43,7 @@ #define OPENCV_TLD_DETECTOR #include "precomp.hpp" +#include "opencl_kernels_tracking.hpp" #include "tldEnsembleClassifier.hpp" #include "tldUtils.hpp" @@ -74,9 +75,13 @@ namespace cv inline double ensembleClassifierNum(const uchar* data); inline void prepareClassifiers(int rowstep); double Sr(const Mat_& patch); + double ocl_Sr(const Mat_& patch); double Sc(const Mat_& patch); + double ocl_Sc(const Mat_& patch); std::vector classifiers; + Mat *posExp, *negExp; + int *posNum, *negNum; std::vector > *positiveExamples, *negativeExamples; std::vector *timeStampsPositive, *timeStampsNegative; double *originalVariancePtr; @@ -88,6 +93,7 @@ namespace cv bool isObject, shouldBeIntegrated; }; bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); + bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); protected: diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 2bc10a76b..6c926d706 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -56,7 +56,16 @@ namespace cv detector = Ptr(new TLDDetector()); //Propagate data to Detector - detector->positiveExamples = &positiveExamples; + posNum = 0; + negNum = 0; + posExp = Mat(Size(225, 500), CV_8UC1); + negExp = Mat(Size(225, 500), CV_8UC1); + detector->posNum = &posNum; + detector->negNum = &negNum; + detector->posExp = &posExp; + detector->negExp = &negExp; + + detector->positiveExamples = &positiveExamples; detector->negativeExamples = &negativeExamples; detector->timeStampsPositive = &timeStampsPositive; detector->timeStampsNegative = &timeStampsNegative; @@ -78,6 +87,7 @@ namespace cv //Generate initial positive samples and put them to the model positiveExamples.reserve(200); + for (int i = 0; i < (int)closest.size(); i++) { for (int j = 0; j < 20; j++) @@ -239,12 +249,30 @@ namespace cv std::vector* proxyT; if (positive) { + if (posNum < 500) + { + uchar *patchPtr = example.data; + uchar *modelPtr = posExp.data; + for (int i = 0; i < STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE; i++) + modelPtr[posNum*STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE + i] = patchPtr[i]; + posNum++; + } + proxyV = &positiveExamples; proxyN = &timeStampPositiveNext; proxyT = &timeStampsPositive; } else { + if (negNum < 500) + { + uchar *patchPtr = example.data; + uchar *modelPtr = negExp.data; + for (int i = 0; i < STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE; i++) + modelPtr[negNum*STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE + i] = patchPtr[i]; + negNum++; + } + proxyV = &negativeExamples; proxyN = &timeStampNegativeNext; proxyT = &timeStampsNegative; diff --git a/modules/tracking/src/tldModel.hpp b/modules/tracking/src/tldModel.hpp index 3a1e5149a..67746745b 100644 --- a/modules/tracking/src/tldModel.hpp +++ b/modules/tracking/src/tldModel.hpp @@ -67,6 +67,8 @@ namespace cv Ptr detector; std::vector > positiveExamples, negativeExamples; + Mat posExp, negExp; + int posNum, negNum; std::vector timeStampsPositive, timeStampsNegative; int timeStampPositiveNext, timeStampNegativeNext; double originalVariance_; From 37e73bcb5751854ce2e7393abae18f1d60fb4f19 Mon Sep 17 00:00:00 2001 From: Aakash Chopra Date: Mon, 22 Jun 2015 15:45:58 -0500 Subject: [PATCH 35/98] Fix for https://github.com/Itseez/opencv_contrib/issues/278 - core dump in the case of no match results. Modified reported poses by constraining to the number of poses found. --- .../samples/ppf_load_match.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/surface_matching/samples/ppf_load_match.cpp b/modules/surface_matching/samples/ppf_load_match.cpp index 303e8080b..4837d31ff 100644 --- a/modules/surface_matching/samples/ppf_load_match.cpp +++ b/modules/surface_matching/samples/ppf_load_match.cpp @@ -110,9 +110,22 @@ int main(int argc, char** argv) tick2 = cv::getTickCount(); cout << endl << "PPF Elapsed Time " << (tick2-tick1)/cv::getTickFrequency() << " sec" << endl; - - // Get only first N results - int N = 2; + + //check results size from match call above + ulong results_size = results.size(); + cout << "Number of matching poses: " << results_size; + if (results_size == 0) { + cout << endl << "No matching poses found. Exiting." << endl; + exit(0); + } + + // Get only first N results - but adjust to results size if num of results are less than that specified by N + ulong N = 2; + if (results_size < N) { + cout << endl << "Reducing matching poses to be reported (as specified in code): " + << N << " to the number of matches found: " << results_size << endl; + N = results_size; + } vector resultsSub(results.begin(),results.begin()+N); // Create an instance of ICP From 8e52a098e9248c3dff359f5671962887dcf9aab2 Mon Sep 17 00:00:00 2001 From: Aakash Chopra Date: Mon, 22 Jun 2015 16:20:35 -0500 Subject: [PATCH 36/98] ulong -> size_t --- modules/surface_matching/samples/ppf_load_match.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/surface_matching/samples/ppf_load_match.cpp b/modules/surface_matching/samples/ppf_load_match.cpp index 4837d31ff..975c242a5 100644 --- a/modules/surface_matching/samples/ppf_load_match.cpp +++ b/modules/surface_matching/samples/ppf_load_match.cpp @@ -112,7 +112,7 @@ int main(int argc, char** argv) (tick2-tick1)/cv::getTickFrequency() << " sec" << endl; //check results size from match call above - ulong results_size = results.size(); + size_t results_size = results.size(); cout << "Number of matching poses: " << results_size; if (results_size == 0) { cout << endl << "No matching poses found. Exiting." << endl; @@ -120,7 +120,7 @@ int main(int argc, char** argv) } // Get only first N results - but adjust to results size if num of results are less than that specified by N - ulong N = 2; + size_t N = 2; if (results_size < N) { cout << endl << "Reducing matching poses to be reported (as specified in code): " << N << " to the number of matches found: " << results_size << endl; From 261086b132a541b6a4c3201905b2ce329ee88a2d Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 24 Jun 2015 11:43:39 +0900 Subject: [PATCH 37/98] Use Doxigen format --- .../include/opencv2/tracking/tracker.hpp | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 8de034d16..25d0feb30 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1202,21 +1202,29 @@ class CV_EXPORTS_W TrackerKCF : public Tracker struct CV_EXPORTS Params { Params(); + + /** + * \brief Read parameters from file, currently unused + */ void read( const FileNode& /*fn*/ ); + + /** + * \brief Read parameters from file, currently unused + */ void write( FileStorage& /*fs*/ ) const; - double sigma; // gaussian kernel bandwidth - double lambda; // regularization - double interp_factor; // linear interpolation factor for adaptation - double output_sigma_factor; // spatial bandwidth (proportional to target) - double pca_learning_rate; // compression learning rate - bool resize; // activate the resize feature to improve the processing speed - bool splitCoeff; // split the training coefficients into two matrices - bool wrapKernel; // wrap around the kernel values - bool compressFeature; // activate pca method to compress the features - int max_patch_size; // threshold for the ROI size - int compressed_size; // feature size after compression - MODE descriptor; // descriptor type + double sigma; //!< gaussian kernel bandwidth + double lambda; //!< regularization + double interp_factor; //!< linear interpolation factor for adaptation + double output_sigma_factor; //!< spatial bandwidth (proportional to target) + double pca_learning_rate; //!< compression learning rate + bool resize; //!< activate the resize feature to improve the processing speed + bool splitCoeff; //!< split the training coefficients into two matrices + bool wrapKernel; //!< wrap around the kernel values + bool compressFeature; //!< activate pca method to compress the features + int max_patch_size; //!< threshold for the ROI size + int compressed_size; //!< feature size after compression + MODE descriptor; //!< descriptor type }; /** @brief Constructor From 660174e87925d4dd707e4238dd02af654da2ce8b Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 24 Jun 2015 14:52:36 +0900 Subject: [PATCH 38/98] Remove whitespaces --- modules/tracking/doc/tracking.bib | 16 +- .../include/opencv2/tracking/tracker.hpp | 12 +- modules/tracking/src/trackerKCF.cpp | 204 +++++++++--------- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/modules/tracking/doc/tracking.bib b/modules/tracking/doc/tracking.bib index 3f6d3be55..6b8b46273 100644 --- a/modules/tracking/doc/tracking.bib +++ b/modules/tracking/doc/tracking.bib @@ -83,13 +83,13 @@ year = {2012}, } -@INPROCEEDINGS{KCF_CN, - author={Danelljan, M. and Khan, F.S. and Felsberg, M. and van de Weijer, J.}, - booktitle={Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on}, - title={Adaptive Color Attributes for Real-Time Visual Tracking}, - year={2014}, - month={June}, - pages={1090-1097}, - keywords={computer vision;feature extraction;image colour analysis;image representation;image sequences;adaptive color attributes;benchmark color sequences;color features;color representations;computer vision;image description;real-time visual tracking;tracking-by-detection framework;Color;Computational modeling;Covariance matrices;Image color analysis;Kernel;Target tracking;Visualization;Adaptive Dimensionality Reduction;Appearance Model;Color Features;Visual Tracking}, +@INPROCEEDINGS{KCF_CN, + author={Danelljan, M. and Khan, F.S. and Felsberg, M. and van de Weijer, J.}, + booktitle={Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on}, + title={Adaptive Color Attributes for Real-Time Visual Tracking}, + year={2014}, + month={June}, + pages={1090-1097}, + keywords={computer vision;feature extraction;image colour analysis;image representation;image sequences;adaptive color attributes;benchmark color sequences;color features;color representations;computer vision;image description;real-time visual tracking;tracking-by-detection framework;Color;Computational modeling;Covariance matrices;Image color analysis;Kernel;Target tracking;Visualization;Adaptive Dimensionality Reduction;Appearance Model;Color Features;Visual Tracking}, doi={10.1109/CVPR.2014.143}, } diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 25d0feb30..94c832d95 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1191,8 +1191,8 @@ class CV_EXPORTS_W TrackerTLD : public Tracker /** @brief KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. * This tracking method is an implementation of @cite KCF_ECCV which is extended to KFC with color-names features (@cite KCF_CN). - * The original paper of KCF is available at - * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to * . */ class CV_EXPORTS_W TrackerKCF : public Tracker @@ -1202,21 +1202,21 @@ class CV_EXPORTS_W TrackerKCF : public Tracker struct CV_EXPORTS Params { Params(); - + /** * \brief Read parameters from file, currently unused */ void read( const FileNode& /*fn*/ ); - + /** * \brief Read parameters from file, currently unused */ void write( FileStorage& /*fs*/ ) const; - + double sigma; //!< gaussian kernel bandwidth double lambda; //!< regularization double interp_factor; //!< linear interpolation factor for adaptation - double output_sigma_factor; //!< spatial bandwidth (proportional to target) + double output_sigma_factor; //!< spatial bandwidth (proportional to target) double pca_learning_rate; //!< compression learning rate bool resize; //!< activate the resize feature to improve the processing speed bool splitCoeff; //!< split the training coefficients into two matrices diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp index 9fca2880b..58b50014b 100644 --- a/modules/tracking/src/trackerKCF.cpp +++ b/modules/tracking/src/trackerKCF.cpp @@ -65,7 +65,7 @@ namespace cv{ | TrackerKCF |---------------------------*/ namespace cv{ - + /* * Prototype */ @@ -74,23 +74,23 @@ namespace cv{ TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); void read( const FileNode& /*fn*/ ); void write( FileStorage& /*fs*/ ) const; - + protected: /* * basic functions and vars */ bool initImpl( const Mat& /*image*/, const Rect2d& boundingBox ); bool updateImpl( const Mat& image, Rect2d& boundingBox ); - + TrackerKCF::Params params; - + /* * KCF functions and vars */ void createHanningWindow(OutputArray _dst, const cv::Size winSize, const int type) const; void inline fft2(const Mat src, std::vector & dest) const; void inline fft2(const Mat src, Mat & dest) const; - void inline ifft2(const Mat src, Mat & dest) const ; + void inline ifft2(const Mat src, Mat & dest) const; void inline pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB=false) const; void inline sumChannels(std::vector src, Mat & dest) const; void inline updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & _proj_mtx,double pca_rate, int compressed_sz) const; @@ -100,16 +100,16 @@ namespace cv{ void denseGaussKernel(const double sigma, const Mat _x, const Mat _y, Mat & _k) const; void calcResponse(const Mat _alphaf, const Mat _k, Mat & _response) const; void calcResponse(const Mat _alphaf, const Mat _alphaf_den, const Mat _k, Mat & _response) const; - - void shiftRows(Mat& mat) const; + + void shiftRows(Mat& mat) const; void shiftRows(Mat& mat, int n) const; void shiftCols(Mat& mat, int n) const; - + private: double output_sigma; Rect2d roi; Mat hann; //hann window filter - + Mat y,yf; // training response and its FFT Mat x,xf; // observation and its FFT Mat k,kf; // dense gaussian kernel and its FFT @@ -119,12 +119,12 @@ namespace cv{ Mat z, new_z; // model Mat response; // detection result Mat old_cov_mtx, proj_mtx; // for feature compression - + bool resizeImage; // resize the image whenever needed and the patch size is large - + int frame; }; - + /* * Constructor */ @@ -136,10 +136,10 @@ namespace cv{ { isInit = false; resizeImage = false; - + CV_Assert(params.descriptor == GRAY || params.descriptor == CN /*|| params.descriptor == CN2*/); } - + void TrackerKCFImpl::read( const cv::FileNode& fn ){ params.read( fn ); } @@ -147,9 +147,9 @@ namespace cv{ void TrackerKCFImpl::write( cv::FileStorage& fs ) const { params.write( fs ); } - + /* - * Initialization: + * Initialization: * - creating hann window filter * - ROI padding * - creating a gaussian response for the training ground-truth @@ -158,11 +158,11 @@ namespace cv{ bool TrackerKCFImpl::initImpl( const Mat& /*image*/, const Rect2d& boundingBox ){ frame=0; roi = boundingBox; - + //calclulate output sigma output_sigma=sqrt(roi.width*roi.height)*params.output_sigma_factor; output_sigma=-0.5/(output_sigma*output_sigma); - + //resize the ROI whenever needed if(params.resize && roi.width*roi.height>params.max_patch_size){ resizeImage=true; @@ -170,21 +170,21 @@ namespace cv{ roi.y/=2.0; roi.width/=2.0; roi.height/=2.0; - } - + } + // add padding to the roi roi.x-=roi.width/2; roi.y-=roi.height/2; roi.width*=2; roi.height*=2; - + // initialize the hann window filter createHanningWindow(hann, roi.size(), CV_64F); if(params.descriptor==CN){ Mat layers[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; - merge(layers, 10, hann); + merge(layers, 10, hann); } - + // create gaussian response y=Mat::zeros((int)roi.height,(int)roi.width,CV_64F); for(unsigned i=0;i(i,j)=(i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1); } } - + y*=(double)output_sigma; cv::exp(y,y); - + // perform fourier transfor to the gaussian response fft2(y,yf); - + model=Ptr(new TrackerKCFModel(params)); - + // TODO: return true only if roi inside the image return true; } - + /* * Main part of the KCF algorithm */ @@ -212,17 +212,17 @@ namespace cv{ double minVal, maxVal; // min-max response Point minLoc,maxLoc; // min-max location Mat zc; - + Mat img=image.clone(); // check the channels of the input image, grayscale is preferred CV_Assert(image.channels() == 1 || image.channels() == 3); - + // resize the image whenever needed if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2)); - + // extract and pre-process the patch if(!getSubWindow(img,roi, x))return false; - + // detection part if(frame>0){ //compute the gaussian kernel @@ -232,52 +232,52 @@ namespace cv{ denseGaussKernel(params.sigma,x,zc,k); }else denseGaussKernel(params.sigma,x,z,k); - + // calculate filter response if(params.splitCoeff) calcResponse(alphaf,alphaf_den,k,response); else calcResponse(alphaf,k,response); - + // extract the maximum response minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); roi.x+=(maxLoc.x-roi.width/2+1); roi.y+=(maxLoc.y-roi.height/2+1); - + // update the bounding box boundingBox.x=(resizeImage?roi.x*2:roi.x)+boundingBox.width/2; boundingBox.y=(resizeImage?roi.y*2:roi.y)+boundingBox.height/2; } - + // extract the patch for learning purpose if(!getSubWindow(img,roi, x))return false; - + //update the training data new_z=x.clone(); if(frame==0) z=x.clone(); else - z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; - + z=(1.0-params.interp_factor)*z+params.interp_factor*new_z; + if(params.compressFeature){ // feature compression updateProjectionMatrix(z,old_cov_mtx,proj_mtx,params.pca_learning_rate,params.compressed_size); compress(proj_mtx,x,x); } - + // Kernel Regularized Least-Squares, calculate alphas denseGaussKernel(params.sigma,x,x,k); - + fft2(k,kf); kf_lambda=kf+params.lambda; - + /* TODO: optimize this element-wise division * new_alphaf=yf./kf * z=(a+bi)/(c+di)[(ac+bd)+i(bc-ad)]/(c^2+d^2) - */ - new_alphaf=Mat_(yf.rows, yf.cols); + */ + new_alphaf=Mat_(yf.rows, yf.cols); std::complex temp; - + if(params.splitCoeff){ mulSpectrums(yf,kf,new_alphaf,0); mulSpectrums(kf,kf_lambda,new_alphaf_den,0); @@ -290,7 +290,7 @@ namespace cv{ } } } - + // update the RLS model if(frame==0){ alphaf=new_alphaf.clone(); @@ -299,17 +299,17 @@ namespace cv{ alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; if(params.splitCoeff)alphaf_den=(1.0-params.interp_factor)*alphaf_den+params.interp_factor*new_alphaf_den; } - + frame++; return true; } - - + + /*------------------------------------- | implementation of the KCF functions |-------------------------------------*/ - - /* + + /* * hann window filter */ void TrackerKCFImpl::createHanningWindow(OutputArray _dst, const cv::Size winSize, const int type) const { @@ -346,30 +346,30 @@ namespace cv{ // perform batch sqrt for SSE performance gains //cv::sqrt(dst, dst); //matlab do not use the square rooted version } - + /* * simplification of fourier transform function in opencv */ void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest) const { std::vector layers(src.channels()); std::vector outputs(src.channels()); - + split(src, layers); - + for(int i=0;i & dest) const { std::vector layers(src.channels()); dest.clear(); dest.resize(src.channels()); - + split(src, layers); - + for(int i=0;i src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const { dest.clear(); dest.resize(src1.size()); - + for(unsigned i=0;i layers(src.channels()); std::vector average(src.channels()); split(src,layers); - + for (int i=0;i(i,i)=w.at(i); - } - + } + // update the covariance matrix old_cov=(1.0-pca_rate)*old_cov+pca_rate*_proj_mtx*proj_vars*_proj_mtx.t(); } - + /* * compress the features */ @@ -451,22 +451,22 @@ namespace cv{ Mat compressed=data*_proj_mtx; dest=compressed.reshape(_proj_mtx.cols,src.rows).clone(); } - + /* * obtain the patch and apply hann window filter to it */ bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& patch) const { Rect region=_roi; - + // return false if roi is outside the image if((_roi.x+_roi.width<0) ||(_roi.y+_roi.height<0) ||(_roi.x>=img.cols) ||(_roi.y>=img.rows) )return false; - - // extract patch inside the image + + // extract patch inside the image if(_roi.x<0){region.x=0;region.width+=_roi.x;} if(_roi.y<0){region.y=0;region.height+=_roi.y;} if(_roi.x+_roi.width>img.cols)region.width=img.cols-_roi.x; @@ -475,7 +475,7 @@ namespace cv{ if(region.height>img.rows)region.height=img.rows; patch=img(region).clone(); - + // add some padding to compensate when the patch is outside image border int addTop,addBottom, addLeft, addRight; addTop=region.y-_roi.y; @@ -485,7 +485,7 @@ namespace cv{ copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); if(patch.rows==0 || patch.cols==0)return false; - + // extract the desired descriptors switch(params.descriptor){ case GRAY: @@ -501,21 +501,21 @@ namespace cv{ if(patch.channels()>1)cvtColor(patch,patch, CV_BGR2GRAY); break; } - + patch=patch.mul(hann); // hann window filter - + return true; - + } - + /* Convert BGR to ColorNames */ void TrackerKCFImpl::extractCN(Mat _patch, Mat & cnFeatures) const { Vec3b & pixel = _patch.at(0,0); unsigned index; - + Mat temp = Mat::zeros(_patch.rows,_patch.cols,CV_64FC(10)); - + for(int i=0;i<_patch.rows;i++){ for(int j=0;j<_patch.cols;j++){ pixel=_patch.at(i,j); @@ -527,10 +527,10 @@ namespace cv{ } } } - + cnFeatures=temp.clone(); } - + /* * dense gauss kernel function */ @@ -538,24 +538,24 @@ namespace cv{ std::vector _xf,_yf,xyf_v; Mat xy,xyf; double normX, normY; - + fft2(_x,_xf); fft2(_y,_yf); - + normX=norm(_x); normX*=normX; normY=norm(_y); normY*=normY; - + pixelWiseMult(_xf,_yf,xyf_v,0,true); sumChannels(xyf_v,xyf); ifft2(xyf,xyf); - + if(params.wrapKernel){ shiftRows(xyf, _x.rows/2); shiftCols(xyf, _x.cols/2); } - + //(xx + yy - 2 * xy) / numel(x) xy=(normX+normY-2*xyf)/(_x.rows*_x.cols*_x.channels()); @@ -566,13 +566,13 @@ namespace cv{ if(xy.at(i,j)<0.0)xy.at(i,j)=0.0; } } - + double sig=-1.0/(sigma*sigma); xy=sig*xy; exp(xy,_k); } - + /* CIRCULAR SHIFT Function * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv */ @@ -601,7 +601,7 @@ namespace cv{ shiftRows(mat); } flip(mat,mat,0); - }else{ + }else{ for(int _k=0; _k < n;_k++) { shiftRows(mat); } @@ -628,14 +628,14 @@ namespace cv{ * calculate the detection response */ void TrackerKCFImpl::calcResponse(const Mat _alphaf, const Mat _k, Mat & _response) const { - //alpha f--> 2channels ; k --> 1 channel; + //alpha f--> 2channels ; k --> 1 channel; Mat _kf; fft2(_k,_kf); Mat spec; mulSpectrums(_alphaf,_kf,spec,0,false); ifft2(spec,_response); } - + /* * calculate the detection response for splitted form */ @@ -643,11 +643,11 @@ namespace cv{ Mat _kf; fft2(_k,_kf); Mat spec; - Mat spec2=Mat_(_k.rows, _k.cols); + Mat spec2=Mat_(_k.rows, _k.cols); std::complex temp; - + mulSpectrums(_alphaf,_kf,spec,0,false); - + for(int i=0;i<_k.rows;i++){ for(int j=0;j<_k.cols;j++){ temp=std::complex(spec.at(i,j)[0],spec.at(i,j)[1])/(std::complex(_alphaf_den.at(i,j)[0],_alphaf_den.at(i,j)[1])/*+std::complex(0.0000000001,0.0000000001)*/); @@ -655,11 +655,11 @@ namespace cv{ spec2.at(i,j)[1]=temp.imag(); } } - + ifft2(spec2,_response); } /*----------------------------------------------------------------------*/ - + /* * Parameters */ @@ -673,7 +673,7 @@ namespace cv{ descriptor=CN; splitCoeff=true; wrapKernel=false; - + //feature compression compressFeature=true; compressed_size=2; @@ -683,5 +683,5 @@ namespace cv{ void TrackerKCF::Params::read( const cv::FileNode& /*fn*/ ){} void TrackerKCF::Params::write( cv::FileStorage& /*fs*/ ) const{} - + } /* namespace cv */ From c43dd7d32b57c689f8643cafe2419f2de23025b5 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Wed, 24 Jun 2015 15:00:21 +0900 Subject: [PATCH 39/98] Removing whitespaces in featureColorName.cpp --- modules/tracking/src/featureColorName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index 8f0a42aa7..58b91ce03 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -32813,5 +32813,5 @@ namespace cv{ {0.0030858,-0.016151,0.013017,0.0072284,-0.53357,0.30985,0.0041336,-0.012531,0.00142,-0.33842}, {0.0087778,-0.015645,0.004769,0.011785,-0.54199,0.31505,0.00020476,-0.020282,0.00021236,-0.34675} }; - + } \ No newline at end of file From 374f435378447a6ba33f8df8a9ae0488efe9d213 Mon Sep 17 00:00:00 2001 From: Kurnianggoro Date: Thu, 25 Jun 2015 10:14:51 +0900 Subject: [PATCH 40/98] Add an example code for the KCF tracker --- modules/tracking/samples/kcf.cpp | 162 +++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 modules/tracking/samples/kcf.cpp diff --git a/modules/tracking/samples/kcf.cpp b/modules/tracking/samples/kcf.cpp new file mode 100644 index 000000000..184c53a46 --- /dev/null +++ b/modules/tracking/samples/kcf.cpp @@ -0,0 +1,162 @@ +/*---------------------------------------------- + * Usage: + * example_tracking_kcf