From 2fda95b80f44894be78c49ec6a888af63f9d3576 Mon Sep 17 00:00:00 2001 From: Alex Leontiev Date: Sun, 22 Sep 2013 21:30:54 +0800 Subject: [PATCH] The implementation of particle filtering tracker This contribution aims to implement the sampler based on particle filtering within a generic tracking API that opencv has. It still remains to write the documentation. --- README.md | 6 +- modules/tracking/CMakeLists.txt | 2 +- modules/tracking/doc/tracker_algorithms.rst | 66 +++--- .../include/opencv2/tracking/tracker.hpp | 21 +- modules/tracking/src/PFSolver.hpp | 219 ++++++++++++++++++ modules/tracking/src/TrackingFunctionPF.hpp | 98 ++++++++ .../tracking/src/trackerSamplerAlgorithm.cpp | 32 +++ 7 files changed, 406 insertions(+), 38 deletions(-) create mode 100644 modules/tracking/src/PFSolver.hpp create mode 100644 modules/tracking/src/TrackingFunctionPF.hpp diff --git a/README.md b/README.md index 52a74f8ab..9b7440ed9 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Here is the CMake command for you: ``` $ cd -$ cmake -DOPENCV_EXTRA_MODULES_PATH= +$ cmake -DOPENCV_EXTRA_MODULES_PATH=/modules $ make -j5 ``` @@ -27,5 +27,5 @@ modules from `opencv_contrib` repository. If you don't want all of the modules, use CMake's `BUILD_opencv_*` options. Like in this example: ``` -$ cmake -DOPENCV_EXTRA_MODULES_PATH= -DBUILD_opencv_legacy=OFF -``` \ No newline at end of file +$ cmake -DOPENCV_EXTRA_MODULES_PATH=/modules -DBUILD_opencv_legacy=OFF +``` diff --git a/modules/tracking/CMakeLists.txt b/modules/tracking/CMakeLists.txt index 39983045b..52bd3d0b7 100644 --- a/modules/tracking/CMakeLists.txt +++ b/modules/tracking/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Tracking API") -ocv_define_module(tracking opencv_imgproc) +ocv_define_module(tracking opencv_imgproc opencv_optim) diff --git a/modules/tracking/doc/tracker_algorithms.rst b/modules/tracking/doc/tracker_algorithms.rst index 519fb4219..b8fed894a 100644 --- a/modules/tracking/doc/tracker_algorithms.rst +++ b/modules/tracking/doc/tracker_algorithms.rst @@ -3,7 +3,7 @@ Tracker Algorithms .. highlight:: cpp -Two algorithms will be implemented soon, the first is MIL (Multiple Instance Learning) [MIL]_ and second is Online Boosting [OLB]_. +The following algorithms are implemented at the moment. .. [MIL] B Babenko, M-H Yang, and S Belongie, Visual Tracking with Online Multiple Instance Learning, In CVPR, 2009 @@ -13,7 +13,8 @@ TrackerBoosting --------------- This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm. -The classifier uses the surrounding background as negative examples in update step to avoid the drifting problem. +The classifier uses the surrounding background as negative examples in update step to avoid the drifting problem. The implementation is based on +[OLB]_. .. ocv:class:: TrackerBoosting @@ -33,10 +34,39 @@ Implementation of TrackerBoosting from :ocv:class:`Tracker`:: }; +TrackerBoosting::Params +----------------------------------------------------------------------- + +.. ocv:struct:: TrackerBoosting::Params + +List of BOOSTING parameters:: + + struct CV_EXPORTS Params + { + Params(); + int numClassifiers; //the number of classifiers to use in a OnlineBoosting algorithm + float samplerOverlap; //search region parameters to use in a OnlineBoosting algorithm + float samplerSearchFactor; // search region parameters to use in a OnlineBoosting algorithm + int iterationInit; //the initial iterations + int featureSetNumFeatures; // #features + + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + }; + +TrackerBoosting::TrackerBoosting +----------------------------------------------------------------------- + +Constructor + +.. ocv:function:: bool TrackerBoosting::TrackerBoosting( const TrackerBoosting::Params ¶meters = TrackerBoosting::Params() ) + + :param parameters: BOOSTING parameters :ocv:struct:`TrackerBoosting::Params` + TrackerMIL ---------- -The MIL algorithm trains a classifier in an online manner to separate the object from the background. Multiple Instance Learning avoids the drift problem for a robust tracking. +The MIL algorithm trains a classifier in an online manner to separate the object from the background. Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is based on [MIL]_. Original code can be found here http://vision.ucsd.edu/~bbabenko/project_miltrack.shtml @@ -89,33 +119,3 @@ Constructor .. ocv:function:: bool TrackerMIL::TrackerMIL( const TrackerMIL::Params ¶meters = TrackerMIL::Params() ) :param parameters: MIL parameters :ocv:struct:`TrackerMIL::Params` - - -TrackerBoosting::Params ------------------- - -.. ocv:struct:: TrackerBoosting::Params - -List of BOOSTING parameters:: - - struct CV_EXPORTS Params - { - Params(); - int numClassifiers; //the number of classifiers to use in a OnlineBoosting algorithm - float samplerOverlap; //search region parameters to use in a OnlineBoosting algorithm - float samplerSearchFactor; // search region parameters to use in a OnlineBoosting algorithm - int iterationInit; //the initial iterations - int featureSetNumFeatures; // #features - - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; - -TrackerBoosting::TrackerBoosting ----------------------- - -Constructor - -.. ocv:function:: bool TrackerBoosting::TrackerBoosting( const TrackerBoosting::Params ¶meters = TrackerBoosting::Params() ) - - :param parameters: BOOSTING parameters :ocv:struct:`TrackerBoosting::Params` diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index bea3e8558..7f5886dd7 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -47,6 +47,7 @@ #include "feature.hpp" #include "onlineMIL.hpp" #include "onlineBoosting.hpp" +#include "opencv2/optim.hpp" #include /* @@ -796,6 +797,25 @@ class CV_EXPORTS_W TrackerSamplerCS : public TrackerSamplerAlgorithm }; +class CV_EXPORTS_W TrackerSamplerPF : public TrackerSamplerAlgorithm{ +public: + struct CV_EXPORTS Params + { + Params(); + int iterationNum; + int particlesNum; + double alpha; + Mat_ std; + }; + TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters = TrackerSamplerPF::Params()); +protected: + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ); +private: + Params params; + Ptr _solver; + Ptr _function; +}; + /************************************ Specific TrackerFeature Classes ************************************/ /** @@ -1015,7 +1035,6 @@ class CV_EXPORTS_W TrackerBoosting : public Tracker Params params; AlgorithmInfo* info() const; }; - } /* namespace cv */ #endif diff --git a/modules/tracking/src/PFSolver.hpp b/modules/tracking/src/PFSolver.hpp new file mode 100644 index 000000000..f85047cee --- /dev/null +++ b/modules/tracking/src/PFSolver.hpp @@ -0,0 +1,219 @@ +#include "opencv2/optim.hpp" +#include "opencv2/core/core_c.h" +#include +#include +#include +#define WEIGHTED + +namespace cv{ + + //!particle filtering class + class PFSolver : public optim::Solver{ + public: + class Function : public optim::Solver::Function + { + public: + //!if parameters have no sense due to some reason (e.g. lie outside of function domain), this function "corrects" them, + //!that is brings to the function domain + virtual void correctParams(double* /*optParams*/)const{} + //!is used when there is a dependence on the number of iterations done in calc(), note that levels are counted starting from 1 + virtual void setLevel(int /*level*/, int /*levelsNum*/){} + }; + PFSolver(); + void getOptParam(OutputArray params)const; + int iteration(); + double minimize(InputOutputArray x); + + void setParticlesNum(int num); + int getParticlesNum(); + void setAlpha(double AlphaM); + double getAlpha(); + void getParamsSTD(OutputArray std)const; + void setParamsSTD(InputArray std); + + Ptr getFunction() const; + void setFunction(const Ptr& f); + TermCriteria getTermCriteria() const; + void setTermCriteria(const TermCriteria& termcrit); + private: + Mat_ _std,_particles,_logweight; + Ptr _Function; + PFSolver::Function* _real_function; + TermCriteria _termcrit; + int _maxItNum,_iter,_particlesNum; + double _alpha; + inline void normalize(Mat_& row); + RNG rng; + }; + + CV_EXPORTS_W Ptr createPFSolver(const Ptr& f=Ptr(),InputArray std=Mat(), + TermCriteria termcrit=TermCriteria(TermCriteria::MAX_ITER,5,0.0),int particlesNum=100,double alpha=0.6); + + PFSolver::PFSolver(){ + _Function=Ptr(); + _real_function=NULL; + _std=Mat_(); + rng=RNG(getTickCount()); + } + void PFSolver::getOptParam(OutputArray params)const{ + params.create(1,_std.rows,CV_64FC1); + Mat mat(1,_std.rows,CV_64FC1); +#ifdef WEIGHTED + mat.setTo(0.0); + for(int i=0;i<_particles.rows;i++){ + mat+=_particles.row(i)/exp(-_logweight(0,i)); + } + _real_function->correctParams((double*)mat.data); + mat.copyTo(params); +#else + params.create(1,_std.rows,CV_64FC1); + Mat optimus=_particles.row(std::max_element(_logweight.begin(),_logweight.end())-_logweight.begin()); + _real_function->correctParams(optimus.data); + optimus.copyTo(params); +#endif + } + int PFSolver::iteration(){ + if(_iter>=_maxItNum){ + return _maxItNum+1; + } + + _real_function->setLevel(_iter+1,_maxItNum); + + //perturb + for(int j=0;j<_particles.cols;j++){ + double sigma=_std(0,j); + for(int i=0;i<_particles.rows;i++){ + _particles(i,j)+=rng.gaussian(sigma); + } + } + + //measure + for(int i=0;i<_particles.rows;i++){ + _real_function->correctParams((double*)_particles.row(i).data); + _logweight(0,i)=-(_real_function->calc((double*)_particles.row(i).data)); + } + //normalize + normalize(_logweight); + //replicate + Mat_ new_particles(_particlesNum,_std.cols); + int num_particles=0; + for(int i=0;i<_particles.rows;i++){ + int num_replicons=cvFloor(new_particles.rows/exp(-_logweight(0,i))); + for(int j=0;j maxrow=_particles.row(std::max_element(_logweight.begin(),_logweight.end())-_logweight.begin()); + for(;num_particles0); + Mat mat_x=x.getMat(); + CV_Assert(mat_x.type()==CV_64FC1 && MIN(mat_x.rows,mat_x.cols)==1 && MAX(mat_x.rows,mat_x.cols)==_std.cols); + + _iter=0; + _particles=Mat_(_particlesNum,_std.cols); + if(mat_x.rows>1){ + mat_x=mat_x.t(); + } + for(int i=0;i<_particles.rows;i++){ + mat_x.copyTo(_particles.row(i)); + } + + _logweight.create(1,_particles.rows); + _logweight.setTo(-log(_particles.rows)); + return 0.0; + } + + void PFSolver::setParticlesNum(int num){ + CV_Assert(num>0); + _particlesNum=num; + } + int PFSolver::getParticlesNum(){ + return _particlesNum; + } + void PFSolver::setAlpha(double AlphaM){ + CV_Assert(0 PFSolver::getFunction() const{ + return _Function; + } + void PFSolver::setFunction(const Ptr& f){ + CV_Assert(f.empty()==false); + + Ptr non_const_f(f); + Solver::Function* f_ptr=static_cast(non_const_f); + + PFSolver::Function *pff=dynamic_cast(f_ptr); + CV_Assert(pff!=NULL); + _Function=f; + _real_function=pff; + } + TermCriteria PFSolver::getTermCriteria() const{ + return TermCriteria(TermCriteria::MAX_ITER,_maxItNum,0.0); + } + void PFSolver::setTermCriteria(const TermCriteria& termcrit){ + CV_Assert(termcrit.type==TermCriteria::MAX_ITER && termcrit.maxCount>0); + _maxItNum=termcrit.maxCount; + } + void PFSolver::getParamsSTD(OutputArray std)const{ + std.create(1,_std.cols,CV_64FC1); + _std.copyTo(std); + } + void PFSolver::setParamsSTD(InputArray std){ + Mat m=std.getMat(); + CV_Assert(MIN(m.cols,m.rows)==1 && m.type()==CV_64FC1); + int ndim=MAX(m.cols,m.rows); + if(ndim!=_std.cols){ + _std=Mat_(1,ndim); + } + if(m.rows==1){ + m.copyTo(_std); + }else{ + Mat std_t=Mat_(ndim,1,(double*)_std.data); + m.copyTo(std_t); + } + } + + Ptr createPFSolver(const Ptr& f,InputArray std,TermCriteria termcrit,int particlesNum,double alpha){ + Ptr ptr(new PFSolver()); + + if(f.empty()==false){ + ptr->setFunction(f); + } + Mat mystd=std.getMat(); + if(mystd.cols!=0 || mystd.rows!=0){ + ptr->setParamsSTD(std); + } + ptr->setTermCriteria(termcrit); + ptr->setParticlesNum(particlesNum); + ptr->setAlpha(alpha); + return ptr; + } + void PFSolver::normalize(Mat_& row){ + double logsum=0.0; + double max=*(std::max_element(row.begin(),row.end())); + row-=max; + for(int i=0;i +#include +#define CLIP(x,a,b) MIN(MAX((x),(a)),(b)) +#define HIST_SIZE 50 + +namespace cv{ + + class TrackingFunctionPF : public PFSolver::Function{ + public: + TrackingFunctionPF(const Mat& chosenRect); + void update(const Mat& image); + double calc(const double* x) const; + void correctParams(double* pt)const; + private: + Mat _image; + static inline Rect rectFromRow(const double* row); + const int _nh,_ns,_nv; + class TrackingHistogram{ + public: + TrackingHistogram(const Mat& img,int nh,int ns,int nv); + double dist(const TrackingHistogram& hist)const; + private: + Mat_ HShist, Vhist; + }; + TrackingHistogram _origHist; + }; + + TrackingFunctionPF::TrackingHistogram::TrackingHistogram(const Mat& img,int nh,int ns,int nv){ + + Mat hsv; + img.convertTo(hsv,CV_32F,1.0/255.0); + cvtColor(hsv,hsv,CV_BGR2HSV); + + HShist=Mat_(nh,ns,0.0); + Vhist=Mat_(1,nv,0.0); + + for(int i=0;i(i,j); + + if(pt.val[1]>0.1 && pt.val[2]>0.2){ + HShist(MIN(nh-1,(int)(nh*pt.val[0]/360.0)),MIN(ns-1,(int)(ns*pt.val[1])))++; + }else{ + Vhist(0,MIN(nv-1,(int)(nv*pt.val[2])))++; + } + }} + + double total=*(sum(HShist)+sum(Vhist)).val; + HShist/=total; + Vhist/=total; + } + double TrackingFunctionPF::TrackingHistogram::dist(const TrackingHistogram& hist)const{ + double res=1.0; + + for(int i=0;ipt[2]){ + double tmp=pt[0]; + pt[0]=pt[2]; + pt[2]=tmp; + } + if(pt[1]>pt[3]){ + double tmp=pt[1]; + pt[1]=pt[3]; + pt[3]=tmp; + } + } + Rect TrackingFunctionPF::rectFromRow(const double* row){ + return Rect(Point_((int)row[0],(int)row[1]),Point_((int)row[2],(int)row[3])); + } +} diff --git a/modules/tracking/src/trackerSamplerAlgorithm.cpp b/modules/tracking/src/trackerSamplerAlgorithm.cpp index 41d2012cb..dce4c87ad 100644 --- a/modules/tracking/src/trackerSamplerAlgorithm.cpp +++ b/modules/tracking/src/trackerSamplerAlgorithm.cpp @@ -41,6 +41,8 @@ #include "precomp.hpp" #include +#include "PFSolver.hpp" +#include "TrackingFunctionPF.hpp" #ifdef _WIN32 #define TIME( arg ) (((double) clock()) / CLOCKS_PER_SEC) @@ -379,4 +381,34 @@ std::vector TrackerSamplerCS::patchesRegularScan( const Mat& image, Rect tr return sample; } +TrackerSamplerPF::Params::Params(){ + iterationNum=20; + particlesNum=100; + alpha=0.9; + std=(Mat_(1,4)<<15.0,15.0,15.0,15.0); +} +TrackerSamplerPF::TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters): + params( parameters ),_function(new TrackingFunctionPF(chosenRect)){ + className="PF"; + _solver=createPFSolver(_function,parameters.std,TermCriteria(TermCriteria::MAX_ITER,parameters.iterationNum,0.0), + parameters.particlesNum,parameters.alpha); +} +bool TrackerSamplerPF::samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ){ + Ptr ptr; + Mat_ _last_guess=(Mat_(1,4)<<(double)boundingBox.x,(double)boundingBox.y, + (double)boundingBox.x+boundingBox.width,(double)boundingBox.y+boundingBox.height); + PFSolver* promoted_solver=dynamic_cast(static_cast(_solver)); + + promoted_solver->setParamsSTD(params.std); + promoted_solver->minimize(_last_guess); + dynamic_cast(static_cast(promoted_solver->getFunction()))->update(image); + while(promoted_solver->iteration() <= promoted_solver->getTermCriteria().maxCount); + promoted_solver->getOptParam(_last_guess); + + Rect res=Rect(Point_((int)_last_guess(0,0),(int)_last_guess(0,1)),Point_((int)_last_guess(0,2),(int)_last_guess(0,3))); + sample.clear(); + sample.push_back(image(res)); + return true; +} + } /* namespace cv */