From 2cb3f6513185b3a7e7dd1a8c187359f6adb8abb1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 27 Oct 2020 18:05:44 +0000 Subject: [PATCH 1/2] tracking: move/copy files before modification --- modules/tracking/include/opencv2/tracking.hpp | 284 --- .../{tracker.hpp => tracking_internals.hpp} | 0 .../opencv2/tracking/tracking_legacy.hpp | 1548 +++++++++++++++++ .../tracker.legacy.hpp} | 0 .../trackerCSRT.legacy.hpp} | 0 .../trackerKCF.legacy.hpp} | 0 .../trackerMIL.legacy.hpp} | 0 7 files changed, 1548 insertions(+), 284 deletions(-) delete mode 100644 modules/tracking/include/opencv2/tracking.hpp rename modules/tracking/include/opencv2/tracking/{tracker.hpp => tracking_internals.hpp} (100%) create mode 100644 modules/tracking/include/opencv2/tracking/tracking_legacy.hpp rename modules/tracking/src/{tracker.cpp => legacy/tracker.legacy.hpp} (100%) rename modules/tracking/src/{trackerCSRT.cpp => legacy/trackerCSRT.legacy.hpp} (100%) rename modules/tracking/src/{trackerKCF.cpp => legacy/trackerKCF.legacy.hpp} (100%) rename modules/tracking/src/{trackerMIL.cpp => legacy/trackerMIL.legacy.hpp} (100%) diff --git a/modules/tracking/include/opencv2/tracking.hpp b/modules/tracking/include/opencv2/tracking.hpp deleted file mode 100644 index 516f5b979..000000000 --- a/modules/tracking/include/opencv2/tracking.hpp +++ /dev/null @@ -1,284 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#ifndef __OPENCV_TRACKING_HPP__ -#define __OPENCV_TRACKING_HPP__ - -#include "opencv2/core/cvdef.h" - -/** @defgroup tracking Tracking API - -Long-term optical tracking API ------------------------------- - -Long-term optical tracking is an important issue for many computer vision applications in -real world scenario. The development in this area is very fragmented and this API is an unique -interface useful for plug several algorithms and compare them. This work is partially based on -@cite AAM and @cite AMVOT . - -These algorithms start from a bounding box of the target and with their internal representation they -avoid the drift during the tracking. These long-term trackers are able to evaluate online the -quality of the location of the target in the new frame, without ground truth. - -There are three main components: the TrackerSampler, the TrackerFeatureSet and the TrackerModel. The -first component is the object that computes the patches over the frame based on the last target -location. The TrackerFeatureSet is the class that manages the Features, is possible plug many kind -of these (HAAR, HOG, LBP, Feature2D, etc). The last component is the internal representation of the -target, it is the appearance model. It stores all state candidates and compute the trajectory (the -most likely target states). The class TrackerTargetState represents a possible state of the target. -The TrackerSampler and the TrackerFeatureSet are the visual representation of the target, instead -the TrackerModel is the statistical model. - -A recent benchmark between these algorithms can be found in @cite OOT - -Creating Your Own %Tracker --------------------- - -If you want to create a new tracker, here's what you have to do. First, decide on the name of the class -for the tracker (to meet the existing style, we suggest something with prefix "tracker", e.g. -trackerMIL, trackerBoosting) -- we shall refer to this choice as to "classname" in subsequent. - -- Declare your tracker in modules/tracking/include/opencv2/tracking/tracker.hpp. Your tracker should inherit from - Tracker (please, see the example below). You should declare the specialized Param structure, - where you probably will want to put the data, needed to initialize your tracker. You should - get something similar to : -@code - class CV_EXPORTS_W TrackerMIL : public Tracker - { - public: - struct CV_EXPORTS Params - { - Params(); - //parameters for sampler - float samplerInitInRadius; // radius for gathering positive instances during init - int samplerInitMaxNegNum; // # negative samples to use during init - float samplerSearchWinSize; // size of search window - float samplerTrackInRadius; // radius for gathering positive instances during tracking - int samplerTrackMaxPosNum; // # positive samples to use during tracking - int samplerTrackMaxNegNum; // # negative samples to use during tracking - int featureSetNumFeatures; // #features - - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; -@endcode - of course, you can also add any additional methods of your choice. It should be pointed out, - however, that it is not expected to have a constructor declared, as creation should be done via - the corresponding create() method. -- Finally, you should implement the function with signature : -@code - Ptr classname::create(const classname::Params ¶meters){ - ... - } -@endcode - That function can (and probably will) return a pointer to some derived class of "classname", - which will probably have a real constructor. - -Every tracker has three component TrackerSampler, TrackerFeatureSet and TrackerModel. The first two -are instantiated from Tracker base class, instead the last component is abstract, so you must -implement your TrackerModel. - -### TrackerSampler - -TrackerSampler is already instantiated, but you should define the sampling algorithm and add the -classes (or single class) to TrackerSampler. You can choose one of the ready implementation as -TrackerSamplerCSC or you can implement your sampling method, in this case the class must inherit -TrackerSamplerAlgorithm. Fill the samplingImpl method that writes the result in "sample" output -argument. - -Example of creating specialized TrackerSamplerAlgorithm TrackerSamplerCSC : : -@code - class CV_EXPORTS_W TrackerSamplerCSC : public TrackerSamplerAlgorithm - { - public: - TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); - ~TrackerSamplerCSC(); - ... - - protected: - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ); - ... - - }; -@endcode - -Example of adding TrackerSamplerAlgorithm to TrackerSampler : : -@code - //sampler is the TrackerSampler - Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; - - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); -@endcode -@sa - TrackerSamplerCSC, TrackerSamplerAlgorithm - -### TrackerFeatureSet - -TrackerFeatureSet is already instantiated (as first) , but you should define what kinds of features -you'll use in your tracker. You can use multiple feature types, so you can add a ready -implementation as TrackerFeatureHAAR in your TrackerFeatureSet or develop your own implementation. -In this case, in the computeImpl method put the code that extract the features and in the selection -method optionally put the code for the refinement and selection of the features. - -Example of creating specialized TrackerFeature TrackerFeatureHAAR : : -@code - class CV_EXPORTS_W TrackerFeatureHAAR : public TrackerFeature - { - public: - TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); - ~TrackerFeatureHAAR(); - void selection( Mat& response, int npoints ); - ... - - protected: - bool computeImpl( const std::vector& images, Mat& response ); - ... - - }; -@endcode -Example of adding TrackerFeature to TrackerFeatureSet : : -@code - //featureSet is the TrackerFeatureSet - Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); - featureSet->addTrackerFeature( trackerFeature ); -@endcode -@sa - TrackerFeatureHAAR, TrackerFeatureSet - -### TrackerModel - -TrackerModel is abstract, so in your implementation you must develop your TrackerModel that inherit -from TrackerModel. Fill the method for the estimation of the state "modelEstimationImpl", that -estimates the most likely target location, see @cite AAM table I (ME) for further information. Fill -"modelUpdateImpl" in order to update the model, see @cite AAM table I (MU). In this class you can use -the :cConfidenceMap and :cTrajectory to storing the model. The first represents the model on the all -possible candidate states and the second represents the list of all estimated states. - -Example of creating specialized TrackerModel TrackerMILModel : : -@code - class TrackerMILModel : public TrackerModel - { - public: - TrackerMILModel( const Rect& boundingBox ); - ~TrackerMILModel(); - ... - - protected: - void modelEstimationImpl( const std::vector& responses ); - void modelUpdateImpl(); - ... - - }; -@endcode -And add it in your Tracker : : -@code - bool TrackerMIL::initImpl( const Mat& image, const Rect2d& boundingBox ) - { - ... - //model is the general TrackerModel field of the general Tracker - model = new TrackerMILModel( boundingBox ); - ... - } -@endcode -In the last step you should define the TrackerStateEstimator based on your implementation or you can -use one of ready class as TrackerStateEstimatorMILBoosting. It represent the statistical part of the -model that estimates the most likely target state. - -Example of creating specialized TrackerStateEstimator TrackerStateEstimatorMILBoosting : : -@code - class CV_EXPORTS_W TrackerStateEstimatorMILBoosting : public TrackerStateEstimator - { - class TrackerMILTargetState : public TrackerTargetState - { - ... - }; - - public: - TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); - ~TrackerStateEstimatorMILBoosting(); - ... - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ); - void updateImpl( std::vector& confidenceMaps ); - ... - - }; -@endcode -And add it in your TrackerModel : : -@code - //model is the TrackerModel of your Tracker - Ptr stateEstimator = new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ); - model->setTrackerStateEstimator( stateEstimator ); -@endcode -@sa - TrackerModel, TrackerStateEstimatorMILBoosting, TrackerTargetState - -During this step, you should define your TrackerTargetState based on your implementation. -TrackerTargetState base class has only the bounding box (upper-left position, width and height), you -can enrich it adding scale factor, target rotation, etc. - -Example of creating specialized TrackerTargetState TrackerMILTargetState : : -@code - class TrackerMILTargetState : public TrackerTargetState - { - public: - TrackerMILTargetState( const Point2f& position, int targetWidth, int targetHeight, bool foreground, const Mat& features ); - ~TrackerMILTargetState(); - ... - - private: - bool isTarget; - Mat targetFeatures; - ... - - }; -@endcode - -*/ - -#include -#include - -#endif //__OPENCV_TRACKING_HPP__ diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp similarity index 100% rename from modules/tracking/include/opencv2/tracking/tracker.hpp rename to modules/tracking/include/opencv2/tracking/tracking_internals.hpp diff --git a/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp new file mode 100644 index 000000000..3c09c2b77 --- /dev/null +++ b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp @@ -0,0 +1,1548 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#ifndef __OPENCV_TRACKER_HPP__ +#define __OPENCV_TRACKER_HPP__ + +#include "opencv2/core.hpp" +#include "opencv2/imgproc/types_c.h" +#include "feature.hpp" +#include "onlineMIL.hpp" +#include "onlineBoosting.hpp" + +/* + * Partially based on: + * ==================================================================================================================== + * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation + * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking + * + * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml + * + */ + +namespace cv +{ + +//! @addtogroup tracking +//! @{ + +/************************************ TrackerFeature Base Classes ************************************/ + +/** @brief Abstract base class for TrackerFeature that represents the feature. + */ +class CV_EXPORTS TrackerFeature +{ + public: + virtual ~TrackerFeature(); + + /** @brief Compute the features in the images collection + @param images The images + @param response The output response + */ + void compute( const std::vector& images, Mat& response ); + + /** @brief Create TrackerFeature by tracker feature type + @param trackerFeatureType The TrackerFeature name + + The modes available now: + + - "HAAR" -- Haar Feature-based + + The modes that will be available soon: + + - "HOG" -- Histogram of Oriented Gradients features + - "LBP" -- Local Binary Pattern features + - "FEATURE2D" -- All types of Feature2D + */ + static Ptr create( const String& trackerFeatureType ); + + /** @brief Identify most effective features + @param response Collection of response for the specific TrackerFeature + @param npoints Max number of features + + @note This method modifies the response parameter + */ + virtual void selection( Mat& response, int npoints ) = 0; + + /** @brief Get the name of the specific TrackerFeature + */ + String getClassName() const; + + protected: + + virtual bool computeImpl( const std::vector& images, Mat& response ) = 0; + + String className; +}; + +/** @brief Class that manages the extraction and selection of features + +@cite AAM Feature Extraction and Feature Set Refinement (Feature Processing and Feature Selection). +See table I and section III C @cite AMVOT Appearance modelling -\> Visual representation (Table II, +section 3.1 - 3.2) + +TrackerFeatureSet is an aggregation of TrackerFeature + +@sa + TrackerFeature + + */ +class CV_EXPORTS TrackerFeatureSet +{ + public: + + TrackerFeatureSet(); + + ~TrackerFeatureSet(); + + /** @brief Extract features from the images collection + @param images The input images + */ + void extraction( const std::vector& images ); + + /** @brief Identify most effective features for all feature types (optional) + */ + void selection(); + + /** @brief Remove outliers for all feature types (optional) + */ + void removeOutliers(); + + /** @brief Add TrackerFeature in the collection. Return true if TrackerFeature is added, false otherwise + @param trackerFeatureType The TrackerFeature name + + The modes available now: + + - "HAAR" -- Haar Feature-based + + The modes that will be available soon: + + - "HOG" -- Histogram of Oriented Gradients features + - "LBP" -- Local Binary Pattern features + - "FEATURE2D" -- All types of Feature2D + + Example TrackerFeatureSet::addTrackerFeature : : + @code + //sample usage: + + Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); + featureSet->addTrackerFeature( trackerFeature ); + + //or add CSC sampler with default parameters + //featureSet->addTrackerFeature( "HAAR" ); + @endcode + @note If you use the second method, you must initialize the TrackerFeature + */ + bool addTrackerFeature( String trackerFeatureType ); + + /** @overload + @param feature The TrackerFeature class + */ + bool addTrackerFeature( Ptr& feature ); + + /** @brief Get the TrackerFeature collection (TrackerFeature name, TrackerFeature pointer) + */ + const std::vector > >& getTrackerFeature() const; + + /** @brief Get the responses + + @note Be sure to call extraction before getResponses Example TrackerFeatureSet::getResponses : : + */ + const std::vector& getResponses() const; + + private: + + void clearResponses(); + bool blockAddTrackerFeature; + + std::vector > > features; //list of features + std::vector responses; //list of response after compute + +}; + +/************************************ TrackerSampler Base Classes ************************************/ + +/** @brief Abstract base class for TrackerSamplerAlgorithm that represents the algorithm for the specific +sampler. + */ +class CV_EXPORTS TrackerSamplerAlgorithm +{ + public: + /** + * \brief Destructor + */ + virtual ~TrackerSamplerAlgorithm(); + + /** @brief Create TrackerSamplerAlgorithm by tracker sampler type. + @param trackerSamplerType The trackerSamplerType name + + The modes available now: + + - "CSC" -- Current State Center + - "CS" -- Current State + */ + static Ptr create( const String& trackerSamplerType ); + + /** @brief Computes the regions starting from a position in an image. + + Return true if samples are computed, false otherwise + + @param image The current frame + @param boundingBox The bounding box from which regions can be calculated + + @param sample The computed samples @cite AAM Fig. 1 variable Sk + */ + bool sampling( const Mat& image, Rect boundingBox, std::vector& sample ); + + /** @brief Get the name of the specific TrackerSamplerAlgorithm + */ + String getClassName() const; + + protected: + String className; + + virtual bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) = 0; +}; + +/** + * \brief Class that manages the sampler in order to select regions for the update the model of the tracker + * [AAM] Sampling e Labeling. See table I and section III B + */ + +/** @brief Class that manages the sampler in order to select regions for the update the model of the tracker + +@cite AAM Sampling e Labeling. See table I and section III B + +TrackerSampler is an aggregation of TrackerSamplerAlgorithm +@sa + TrackerSamplerAlgorithm + */ +class CV_EXPORTS TrackerSampler +{ + public: + + /** + * \brief Constructor + */ + TrackerSampler(); + + /** + * \brief Destructor + */ + ~TrackerSampler(); + + /** @brief Computes the regions starting from a position in an image + @param image The current frame + @param boundingBox The bounding box from which regions can be calculated + */ + void sampling( const Mat& image, Rect boundingBox ); + + /** @brief Return the collection of the TrackerSamplerAlgorithm + */ + const std::vector > >& getSamplers() const; + + /** @brief Return the samples from all TrackerSamplerAlgorithm, @cite AAM Fig. 1 variable Sk + */ + const std::vector& getSamples() const; + + /** @brief Add TrackerSamplerAlgorithm in the collection. Return true if sampler is added, false otherwise + @param trackerSamplerAlgorithmType The TrackerSamplerAlgorithm name + + The modes available now: + - "CSC" -- Current State Center + - "CS" -- Current State + - "PF" -- Particle Filtering + + Example TrackerSamplerAlgorithm::addTrackerSamplerAlgorithm : : + @code + TrackerSamplerCSC::Params CSCparameters; + Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); + + if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) + return false; + + //or add CSC sampler with default parameters + //sampler->addTrackerSamplerAlgorithm( "CSC" ); + @endcode + @note If you use the second method, you must initialize the TrackerSamplerAlgorithm + */ + bool addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ); + + /** @overload + @param sampler The TrackerSamplerAlgorithm + */ + bool addTrackerSamplerAlgorithm( Ptr& sampler ); + + private: + std::vector > > samplers; + std::vector samples; + bool blockAddTrackerSampler; + + void clearSamples(); +}; + +/************************************ TrackerModel Base Classes ************************************/ + +/** @brief Abstract base class for TrackerTargetState that represents a possible state of the target. + +See @cite AAM \f$\hat{x}^{i}_{k}\f$ all the states candidates. + +Inherits this class with your Target state, In own implementation you can add scale variation, +width, height, orientation, etc. + */ +class CV_EXPORTS TrackerTargetState +{ + public: + virtual ~TrackerTargetState() + { + } + ; + /** + * \brief Get the position + * \return The position + */ + Point2f getTargetPosition() const; + + /** + * \brief Set the position + * \param position The position + */ + void setTargetPosition( const Point2f& position ); + /** + * \brief Get the width of the target + * \return The width of the target + */ + int getTargetWidth() const; + + /** + * \brief Set the width of the target + * \param width The width of the target + */ + void setTargetWidth( int width ); + /** + * \brief Get the height of the target + * \return The height of the target + */ + int getTargetHeight() const; + + /** + * \brief Set the height of the target + * \param height The height of the target + */ + void setTargetHeight( int height ); + + protected: + Point2f targetPosition; + int targetWidth; + int targetHeight; + +}; + +/** @brief Represents the model of the target at frame \f$k\f$ (all states and scores) + +See @cite AAM The set of the pair \f$\langle \hat{x}^{i}_{k}, C^{i}_{k} \rangle\f$ +@sa TrackerTargetState + */ +typedef std::vector, float> > ConfidenceMap; + +/** @brief Represents the estimate states for all frames + +@cite AAM \f$x_{k}\f$ is the trajectory of the target up to time \f$k\f$ + +@sa TrackerTargetState + */ +typedef std::vector > Trajectory; + +/** @brief Abstract base class for TrackerStateEstimator that estimates the most likely target state. + +See @cite AAM State estimator + +See @cite AMVOT Statistical modeling (Fig. 3), Table III (generative) - IV (discriminative) - V (hybrid) + */ +class CV_EXPORTS TrackerStateEstimator +{ + public: + virtual ~TrackerStateEstimator(); + + /** @brief Estimate the most likely target state, return the estimated state + @param confidenceMaps The overall appearance model as a list of :cConfidenceMap + */ + Ptr estimate( const std::vector& confidenceMaps ); + + /** @brief Update the ConfidenceMap with the scores + @param confidenceMaps The overall appearance model as a list of :cConfidenceMap + */ + void update( std::vector& confidenceMaps ); + + /** @brief Create TrackerStateEstimator by tracker state estimator type + @param trackeStateEstimatorType The TrackerStateEstimator name + + The modes available now: + + - "BOOSTING" -- Boosting-based discriminative appearance models. See @cite AMVOT section 4.4 + + The modes available soon: + + - "SVM" -- SVM-based discriminative appearance models. See @cite AMVOT section 4.5 + */ + static Ptr create( const String& trackeStateEstimatorType ); + + /** @brief Get the name of the specific TrackerStateEstimator + */ + String getClassName() const; + + protected: + + virtual Ptr estimateImpl( const std::vector& confidenceMaps ) = 0; + virtual void updateImpl( std::vector& confidenceMaps ) = 0; + String className; +}; + +/** @brief Abstract class that represents the model of the target. It must be instantiated by specialized +tracker + +See @cite AAM Ak + +Inherits this with your TrackerModel + */ +class CV_EXPORTS TrackerModel +{ + public: + + /** + * \brief Constructor + */ + TrackerModel(); + + /** + * \brief Destructor + */ + virtual ~TrackerModel(); + + /** @brief Set TrackerEstimator, return true if the tracker state estimator is added, false otherwise + @param trackerStateEstimator The TrackerStateEstimator + @note You can add only one TrackerStateEstimator + */ + bool setTrackerStateEstimator( Ptr trackerStateEstimator ); + + /** @brief Estimate the most likely target location + + @cite AAM ME, Model Estimation table I + @param responses Features extracted from TrackerFeatureSet + */ + void modelEstimation( const std::vector& responses ); + + /** @brief Update the model + + @cite AAM MU, Model Update table I + */ + void modelUpdate(); + + /** @brief Run the TrackerStateEstimator, return true if is possible to estimate a new state, false otherwise + */ + bool runStateEstimator(); + + /** @brief Set the current TrackerTargetState in the Trajectory + @param lastTargetState The current TrackerTargetState + */ + void setLastTargetState( const Ptr& lastTargetState ); + + /** @brief Get the last TrackerTargetState from Trajectory + */ + Ptr getLastTargetState() const; + + /** @brief Get the list of the ConfidenceMap + */ + const std::vector& getConfidenceMaps() const; + + /** @brief Get the last ConfidenceMap for the current frame + */ + const ConfidenceMap& getLastConfidenceMap() const; + + /** @brief Get the TrackerStateEstimator + */ + Ptr getTrackerStateEstimator() const; + + private: + + void clearCurrentConfidenceMap(); + + protected: + std::vector confidenceMaps; + Ptr stateEstimator; + ConfidenceMap currentConfidenceMap; + Trajectory trajectory; + int maxCMLength; + + virtual void modelEstimationImpl( const std::vector& responses ) = 0; + virtual void modelUpdateImpl() = 0; + +}; + +/************************************ Tracker Base Class ************************************/ + +/** @brief Base abstract class for the long-term tracker: + */ +class CV_EXPORTS_W Tracker : public virtual Algorithm +{ + public: + + virtual ~Tracker() CV_OVERRIDE; + + /** @brief Initialize the tracker with a known bounding box that surrounded the target + @param image The initial frame + @param boundingBox The initial bounding box + + @return True if initialization went succesfully, false otherwise + */ + CV_WRAP bool init( InputArray image, const Rect2d& boundingBox ); + + /** @brief Update the tracker, find the new most likely bounding box for the target + @param image The current frame + @param boundingBox The bounding box that represent the new target location, if true was returned, not + modified otherwise + + @return True means that target was located and false means that tracker cannot locate target in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + CV_WRAP bool update( InputArray image, CV_OUT Rect2d& boundingBox ); + + virtual void read( const FileNode& fn ) CV_OVERRIDE = 0; + virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0; + + protected: + + virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; + virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; + + bool isInit; + + Ptr featureSet; + Ptr sampler; + Ptr model; +}; + + +/************************************ Specific TrackerStateEstimator Classes ************************************/ + +/** @brief TrackerStateEstimator based on Boosting + */ +class CV_EXPORTS TrackerStateEstimatorMILBoosting : public TrackerStateEstimator +{ + public: + + /** + * Implementation of the target state for TrackerStateEstimatorMILBoosting + */ + class TrackerMILTargetState : public TrackerTargetState + { + + public: + /** + * \brief Constructor + * \param position Top left corner of the bounding box + * \param width Width of the bounding box + * \param height Height of the bounding box + * \param foreground label for target or background + * \param features features extracted + */ + TrackerMILTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& features ); + + /** + * \brief Destructor + */ + ~TrackerMILTargetState() + { + } + ; + + /** @brief Set label: true for target foreground, false for background + @param foreground Label for background/foreground + */ + void setTargetFg( bool foreground ); + /** @brief Set the features extracted from TrackerFeatureSet + @param features The features extracted + */ + void setFeatures( const Mat& features ); + /** @brief Get the label. Return true for target foreground, false for background + */ + bool isTargetFg() const; + /** @brief Get the features extracted + */ + Mat getFeatures() const; + + private: + bool isTarget; + Mat targetFeatures; + }; + + /** @brief Constructor + @param nFeatures Number of features for each sample + */ + TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); + ~TrackerStateEstimatorMILBoosting(); + + /** @brief Set the current confidenceMap + @param confidenceMap The current :cConfidenceMap + */ + void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; + void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; + + private: + uint max_idx( const std::vector &v ); + void prepareData( const ConfidenceMap& confidenceMap, Mat& positive, Mat& negative ); + + ClfMilBoost boostMILModel; + bool trained; + int numFeatures; + + ConfidenceMap currentConfidenceMap; +}; + +/** @brief TrackerStateEstimatorAdaBoosting based on ADA-Boosting + */ +class CV_EXPORTS TrackerStateEstimatorAdaBoosting : public TrackerStateEstimator +{ + public: + /** @brief Implementation of the target state for TrackerAdaBoostingTargetState + */ + class TrackerAdaBoostingTargetState : public TrackerTargetState + { + + public: + /** + * \brief Constructor + * \param position Top left corner of the bounding box + * \param width Width of the bounding box + * \param height Height of the bounding box + * \param foreground label for target or background + * \param responses list of features + */ + TrackerAdaBoostingTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& responses ); + + /** + * \brief Destructor + */ + ~TrackerAdaBoostingTargetState() + { + } + ; + + /** @brief Set the features extracted from TrackerFeatureSet + @param responses The features extracted + */ + void setTargetResponses( const Mat& responses ); + /** @brief Set label: true for target foreground, false for background + @param foreground Label for background/foreground + */ + void setTargetFg( bool foreground ); + /** @brief Get the features extracted + */ + Mat getTargetResponses() const; + /** @brief Get the label. Return true for target foreground, false for background + */ + bool isTargetFg() const; + + private: + bool isTarget; + Mat targetResponses; + + }; + + /** @brief Constructor + @param numClassifer Number of base classifiers + @param initIterations Number of iterations in the initialization + @param nFeatures Number of features/weak classifiers + @param patchSize tracking rect + @param ROI initial ROI + */ + TrackerStateEstimatorAdaBoosting( int numClassifer, int initIterations, int nFeatures, Size patchSize, const Rect& ROI ); + + /** + * \brief Destructor + */ + ~TrackerStateEstimatorAdaBoosting(); + + /** @brief Get the sampling ROI + */ + Rect getSampleROI() const; + + /** @brief Set the sampling ROI + @param ROI the sampling ROI + */ + void setSampleROI( const Rect& ROI ); + + /** @brief Set the current confidenceMap + @param confidenceMap The current :cConfidenceMap + */ + void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); + + /** @brief Get the list of the selected weak classifiers for the classification step + */ + std::vector computeSelectedWeakClassifier(); + + /** @brief Get the list of the weak classifiers that should be replaced + */ + std::vector computeReplacedClassifier(); + + /** @brief Get the list of the weak classifiers that replace those to be replaced + */ + std::vector computeSwappedClassifier(); + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; + void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; + + Ptr boostClassifier; + + private: + int numBaseClassifier; + int iterationInit; + int numFeatures; + bool trained; + Size initPatchSize; + Rect sampleROI; + std::vector replacedClassifier; + std::vector swappedClassifier; + + ConfidenceMap currentConfidenceMap; +}; + +/** + * \brief TrackerStateEstimator based on SVM + */ +class CV_EXPORTS TrackerStateEstimatorSVM : public TrackerStateEstimator +{ + public: + TrackerStateEstimatorSVM(); + ~TrackerStateEstimatorSVM(); + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; + void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; +}; + +/************************************ Specific TrackerSamplerAlgorithm Classes ************************************/ + +/** @brief TrackerSampler based on CSC (current state centered), used by MIL algorithm TrackerMIL + */ +class CV_EXPORTS TrackerSamplerCSC : public TrackerSamplerAlgorithm +{ + public: + enum + { + MODE_INIT_POS = 1, //!< mode for init positive samples + MODE_INIT_NEG = 2, //!< mode for init negative samples + MODE_TRACK_POS = 3, //!< mode for update positive samples + MODE_TRACK_NEG = 4, //!< mode for update negative samples + MODE_DETECT = 5 //!< mode for detect samples + }; + + struct CV_EXPORTS Params + { + Params(); + float initInRad; //!< radius for gathering positive instances during init + float trackInPosRad; //!< radius for gathering positive instances during tracking + float searchWinSize; //!< size of search window + int initMaxNegNum; //!< # negative samples to use during init + int trackMaxPosNum; //!< # positive samples to use during training + int trackMaxNegNum; //!< # negative samples to use during training + }; + + /** @brief Constructor + @param parameters TrackerSamplerCSC parameters TrackerSamplerCSC::Params + */ + TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); + + /** @brief Set the sampling mode of TrackerSamplerCSC + @param samplingMode The sampling mode + + The modes are: + + - "MODE_INIT_POS = 1" -- for the positive sampling in initialization step + - "MODE_INIT_NEG = 2" -- for the negative sampling in initialization step + - "MODE_TRACK_POS = 3" -- for the positive sampling in update step + - "MODE_TRACK_NEG = 4" -- for the negative sampling in update step + - "MODE_DETECT = 5" -- for the sampling in detection step + */ + void setMode( int samplingMode ); + + ~TrackerSamplerCSC(); + + protected: + + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; + + private: + + Params params; + int mode; + RNG rng; + + std::vector sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad = 0, int maxnum = 1000000 ); +}; + +/** @brief TrackerSampler based on CS (current state), used by algorithm TrackerBoosting + */ +class CV_EXPORTS TrackerSamplerCS : public TrackerSamplerAlgorithm +{ + public: + enum + { + MODE_POSITIVE = 1, //!< mode for positive samples + MODE_NEGATIVE = 2, //!< mode for negative samples + MODE_CLASSIFY = 3 //!< mode for classify samples + }; + + struct CV_EXPORTS Params + { + Params(); + float overlap; //!& sample ) CV_OVERRIDE; + Rect getROI() const; + private: + Rect getTrackingROI( float searchFactor ); + Rect RectMultiply( const Rect & rect, float f ); + std::vector patchesRegularScan( const Mat& image, Rect trackingROI, Size patchSize ); + void setCheckedROI( Rect imageROI ); + + Params params; + int mode; + Rect trackedPatch; + Rect validROI; + Rect ROI; + +}; + +/** @brief This sampler is based on particle filtering. + +In principle, it can be thought of as performing some sort of optimization (and indeed, this +tracker uses opencv's optim module), where tracker seeks to find the rectangle in given frame, +which is the most *"similar"* to the initial rectangle (the one, given through the constructor). + +The optimization performed is stochastic and somehow resembles genetic algorithms, where on each new +image received (submitted via TrackerSamplerPF::sampling()) we start with the region bounded by +boundingBox, then generate several "perturbed" boxes, take the ones most similar to the original. +This selection round is repeated several times. At the end, we hope that only the most promising box +remaining, and these are combined to produce the subrectangle of image, which is put as a sole +element in array sample. + +It should be noted, that the definition of "similarity" between two rectangles is based on comparing +their histograms. As experiments show, tracker is *not* very succesfull if target is assumed to +strongly change its dimensions. + */ +class CV_EXPORTS TrackerSamplerPF : public TrackerSamplerAlgorithm +{ +public: + /** @brief This structure contains all the parameters that can be varied during the course of sampling + algorithm. Below is the structure exposed, together with its members briefly explained with + reference to the above discussion on algorithm's working. + */ + struct CV_EXPORTS Params + { + Params(); + int iterationNum; //!< number of selection rounds + int particlesNum; //!< number of "perturbed" boxes on each round + double alpha; //!< with each new round we exponentially decrease the amount of "perturbing" we allow (like in simulated annealing) + //!< and this very alpha controls how fast annealing happens, ie. how fast perturbing decreases + Mat_ std; //!< initial values for perturbing (1-by-4 array, as each rectangle is given by 4 values -- coordinates of opposite vertices, + //!< hence we have 4 values to perturb) + }; + /** @brief Constructor + @param chosenRect Initial rectangle, that is supposed to contain target we'd like to track. + @param parameters + */ + TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters = TrackerSamplerPF::Params()); +protected: + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; +private: + Params params; + Ptr _solver; + Ptr _function; +}; + +/************************************ Specific TrackerFeature Classes ************************************/ + +/** + * \brief TrackerFeature based on Feature2D + */ +class CV_EXPORTS TrackerFeatureFeature2d : public TrackerFeature +{ + public: + + /** + * \brief Constructor + * \param detectorType string of FeatureDetector + * \param descriptorType string of DescriptorExtractor + */ + TrackerFeatureFeature2d( String detectorType, String descriptorType ); + + ~TrackerFeatureFeature2d() CV_OVERRIDE; + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + + private: + + std::vector keypoints; +}; + +/** + * \brief TrackerFeature based on HOG + */ +class CV_EXPORTS TrackerFeatureHOG : public TrackerFeature +{ + public: + + TrackerFeatureHOG(); + + ~TrackerFeatureHOG() CV_OVERRIDE; + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + +}; + +/** @brief TrackerFeature based on HAAR features, used by TrackerMIL and many others algorithms +@note HAAR features implementation is copied from apps/traincascade and modified according to MIL + */ +class CV_EXPORTS TrackerFeatureHAAR : public TrackerFeature +{ + public: + struct CV_EXPORTS Params + { + Params(); + int numFeatures; //!< # of rects + Size rectSize; //!< rect size + bool isIntegral; //!< true if input images are integral, false otherwise + }; + + /** @brief Constructor + @param parameters TrackerFeatureHAAR parameters TrackerFeatureHAAR::Params + */ + TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); + + ~TrackerFeatureHAAR() CV_OVERRIDE; + + /** @brief Compute the features only for the selected indices in the images collection + @param selFeatures indices of selected features + @param images The images + @param response Collection of response for the specific TrackerFeature + */ + bool extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ); + + /** @brief Identify most effective features + @param response Collection of response for the specific TrackerFeature + @param npoints Max number of features + + @note This method modifies the response parameter + */ + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + /** @brief Swap the feature in position source with the feature in position target + @param source The source position + @param target The target position + */ + bool swapFeature( int source, int target ); + + /** @brief Swap the feature in position id with the feature input + @param id The position + @param feature The feature + */ + bool swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ); + + /** @brief Get the feature in position id + @param id The position + */ + CvHaarEvaluator::FeatureHaar& getFeatureAt( int id ); + + protected: + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + + private: + + Params params; + Ptr featureEvaluator; +}; + +/** + * \brief TrackerFeature based on LBP + */ +class CV_EXPORTS TrackerFeatureLBP : public TrackerFeature +{ + public: + + TrackerFeatureLBP(); + + ~TrackerFeatureLBP(); + + void selection( Mat& response, int npoints ) CV_OVERRIDE; + + protected: + + bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; + +}; + +/************************************ Specific Tracker Classes ************************************/ + +/** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the +background. + +Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is +based on @cite MIL . + +Original code can be found here + */ +class CV_EXPORTS_W TrackerMIL : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + //parameters for sampler + float samplerInitInRadius; //!< radius for gathering positive instances during init + int samplerInitMaxNegNum; //!< # negative samples to use during init + float samplerSearchWinSize; //!< size of search window + float samplerTrackInRadius; //!< radius for gathering positive instances during tracking + int samplerTrackMaxPosNum; //!< # positive samples to use during tracking + int samplerTrackMaxNegNum; //!< # negative samples to use during tracking + int featureSetNumFeatures; //!< # features + + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + }; + + /** @brief Constructor + @param parameters MIL parameters TrackerMIL::Params + */ + static Ptr create(const TrackerMIL::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerMIL() CV_OVERRIDE {} +}; + +/** @brief the Boosting tracker + +This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm. +The classifier uses the surrounding background as negative examples in update step to avoid the +drifting problem. The implementation is based on @cite OLB . + */ +class CV_EXPORTS_W TrackerBoosting : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + int numClassifiers; //! create(const TrackerBoosting::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerBoosting() CV_OVERRIDE {} +}; + +/** @brief the Median Flow tracker + +Implementation of a paper @cite MedianFlow . + +The tracker is suitable for very smooth and predictable movements when object is visible throughout +the whole sequence. It's quite and accurate for this type of problems (in particular, it was shown +by authors to outperform MIL). During the implementation period the code at +, the courtesy of the author Arthur Amarra, was used for the +reference purpose. + */ +class CV_EXPORTS_W TrackerMedianFlow : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); //! create(const TrackerMedianFlow::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerMedianFlow() CV_OVERRIDE {} +}; + +/** @brief the TLD (Tracking, learning and detection) tracker + +TLD is a novel tracking framework that explicitly decomposes the long-term tracking task into +tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector's +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + */ +class CV_EXPORTS_W TrackerTLD : public Tracker +{ + public: + struct CV_EXPORTS Params + { + Params(); + void read( const FileNode& /*fn*/ ); + void write( FileStorage& /*fs*/ ) const; + }; + + /** @brief Constructor + @param parameters TLD parameters TrackerTLD::Params + */ + static Ptr create(const TrackerTLD::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerTLD() CV_OVERRIDE {} +}; + +/** @brief the KCF (Kernelized Correlation Filter) tracker + + * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. + * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * . + */ +class CV_EXPORTS_W TrackerKCF : public Tracker +{ +public: + /** + * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names + * The modes available now: + - "GRAY" -- Use grayscale values as the feature + - "CN" -- Color-names feature + */ + enum MODE { + GRAY = (1 << 0), + CN = (1 << 1), + CUSTOM = (1 << 2) + }; + + struct CV_EXPORTS Params + { + /** + * \brief Constructor + */ + Params(); + + /** + * \brief Read parameters from a file + */ + void read(const FileNode& /*fn*/); + + /** + * \brief Write parameters to a file + */ + void write(FileStorage& /*fs*/) const; + + float detect_thresh; //!< detection confidence threshold + float sigma; //!< gaussian kernel bandwidth + float lambda; //!< regularization + float interp_factor; //!< linear interpolation factor for adaptation + float output_sigma_factor; //!< spatial bandwidth (proportional to target) + float pca_learning_rate; //!< compression learning rate + bool resize; //!< activate the resize feature to improve the processing speed + bool split_coeff; //!< split the training coefficients into two matrices + bool wrap_kernel; //!< wrap around the kernel values + bool compress_feature; //!< activate the pca method to compress the features + int max_patch_size; //!< threshold for the ROI size + int compressed_size; //!< feature size after compression + int desc_pca; //!< compressed descriptors of TrackerKCF::MODE + int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE + }; + + virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0; + + /** @brief Constructor + @param parameters KCF parameters TrackerKCF::Params + */ + static Ptr create(const TrackerKCF::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerKCF() CV_OVERRIDE {} +}; + +/** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker + + * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, + * GOTURN is much faster due to offline training without online fine-tuning nature. + * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video, + * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly + * robust to viewpoint changes, lighting changes, and deformations. + * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227. + * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2. + * Original paper is here: + * As long as original authors implementation: + * Implementation of training algorithm is placed in separately here due to 3d-party dependencies: + * + * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. +*/ +class CV_EXPORTS_W TrackerGOTURN : public Tracker +{ +public: + struct CV_EXPORTS Params + { + Params(); + void read(const FileNode& /*fn*/); + void write(FileStorage& /*fs*/) const; + String modelTxt; + String modelBin; + }; + + /** @brief Constructor + @param parameters GOTURN parameters TrackerGOTURN::Params + */ + static Ptr create(const TrackerGOTURN::Params ¶meters); + + CV_WRAP static Ptr create(); + + virtual ~TrackerGOTURN() CV_OVERRIDE {} +}; + +/** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker + +The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive Correlation Filters +@note this tracker works with grayscale images, if passed bgr ones, they will get converted internally. +*/ + +class CV_EXPORTS_W TrackerMOSSE : public Tracker +{ + public: + /** @brief Constructor + */ + CV_WRAP static Ptr create(); + + virtual ~TrackerMOSSE() CV_OVERRIDE {} +}; + + +/************************************ MultiTracker Class ---By Laksono Kurnianggoro---) ************************************/ +/** @brief This class is used to track multiple objects using the specified tracker algorithm. + +* The %MultiTracker is naive implementation of multiple object tracking. +* It process the tracked objects independently without any optimization accross the tracked objects. +*/ +class CV_EXPORTS_W MultiTracker : public Algorithm +{ +public: + + /** + * \brief Constructor. + */ + CV_WRAP MultiTracker(); + + /** + * \brief Destructor + */ + ~MultiTracker() CV_OVERRIDE; + + /** + * \brief Add a new object to be tracked. + * + * @param newTracker tracking algorithm to be used + * @param image input image + * @param boundingBox a rectangle represents ROI of the tracked object + */ + CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); + + /** + * \brief Add a set of objects to be tracked. + * @param newTrackers list of tracking algorithms to be used + * @param image input image + * @param boundingBox list of the tracked objects + */ + bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); + + /** + * \brief Update the current tracking status. + * The result will be saved in the internal storage. + * @param image input image + */ + bool update(InputArray image); + + /** + * \brief Update the current tracking status. + * @param image input image + * @param boundingBox the tracking result, represent a list of ROIs of the tracked objects. + */ + CV_WRAP bool update(InputArray image, CV_OUT std::vector & boundingBox); + + /** + * \brief Returns a reference to a storage for the tracked objects, each object corresponds to one tracker algorithm + */ + CV_WRAP const std::vector& getObjects() const; + + /** + * \brief Returns a pointer to a new instance of MultiTracker + */ + CV_WRAP static Ptr create(); + +protected: + //!< storage for the tracker algorithms. + std::vector< Ptr > trackerList; + + //!< storage for the tracked objects, each object corresponds to one tracker algorithm. + std::vector objects; +}; + +/************************************ Multi-Tracker Classes ---By Tyan Vladimir---************************************/ + +/** @brief Base abstract class for the long-term Multi Object Trackers: + +@sa Tracker, MultiTrackerTLD +*/ +class CV_EXPORTS MultiTracker_Alt +{ +public: + /** @brief Constructor for Multitracker + */ + MultiTracker_Alt() + { + targetNum = 0; + } + + /** @brief Add a new target to a tracking-list and initialize the tracker with a known bounding box that surrounded the target + @param image The initial frame + @param boundingBox The initial bounding box of target + @param tracker_algorithm Multi-tracker algorithm + + @return True if new target initialization went succesfully, false otherwise + */ + bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); + + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets + @param image The current frame + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + bool update(InputArray image); + + /** @brief Current number of targets in tracking-list + */ + int targetNum; + + /** @brief Trackers list for Multi-Object-Tracker + */ + std::vector > trackers; + + /** @brief Bounding Boxes list for Multi-Object-Tracker + */ + std::vector boundingBoxes; + /** @brief List of randomly generated colors for bounding boxes display + */ + std::vector colors; +}; + +/** @brief Multi Object %Tracker for TLD. + +TLD is a novel tracking framework that explicitly decomposes +the long-term tracking task into tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector's +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + +@sa Tracker, MultiTracker, TrackerTLD +*/ +class CV_EXPORTS MultiTrackerTLD : public MultiTracker_Alt +{ +public: + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by + optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation + is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20% + + @param image The current frame. + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + bool update_opt(InputArray image); +}; + +/*********************************** CSRT ************************************/ +/** @brief the CSRT tracker + +The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability +*/ +class CV_EXPORTS_W TrackerCSRT : public Tracker +{ +public: + struct CV_EXPORTS Params + { + /** + * \brief Constructor + */ + Params(); + + /** + * \brief Read parameters from a file + */ + void read(const FileNode& /*fn*/); + + /** + * \brief Write parameters to a file + */ + void write(cv::FileStorage& fs) const; + + bool use_hog; + bool use_color_names; + bool use_gray; + bool use_rgb; + bool use_channel_weights; + bool use_segmentation; + + std::string window_function; //!< Window function: "hann", "cheb", "kaiser" + float kaiser_alpha; + float cheb_attenuation; + + float template_size; + float gsl_sigma; + float hog_orientations; + float hog_clip; + float padding; + float filter_lr; + float weights_lr; + int num_hog_channels_used; + int admm_iterations; + int histogram_bins; + float histogram_lr; + int background_ratio; + int number_of_scales; + float scale_sigma_factor; + float scale_model_max_area; + float scale_lr; + float scale_step; + + float psr_threshold; //!< we lost the target, if the psr is lower than this. + }; + + /** @brief Constructor + @param parameters CSRT parameters TrackerCSRT::Params + */ + static Ptr create(const TrackerCSRT::Params ¶meters); + + CV_WRAP static Ptr create(); + + CV_WRAP virtual void setInitialMask(InputArray mask) = 0; + + virtual ~TrackerCSRT() CV_OVERRIDE {} +}; + +//! @} +} /* namespace cv */ + +#endif diff --git a/modules/tracking/src/tracker.cpp b/modules/tracking/src/legacy/tracker.legacy.hpp similarity index 100% rename from modules/tracking/src/tracker.cpp rename to modules/tracking/src/legacy/tracker.legacy.hpp diff --git a/modules/tracking/src/trackerCSRT.cpp b/modules/tracking/src/legacy/trackerCSRT.legacy.hpp similarity index 100% rename from modules/tracking/src/trackerCSRT.cpp rename to modules/tracking/src/legacy/trackerCSRT.legacy.hpp diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/legacy/trackerKCF.legacy.hpp similarity index 100% rename from modules/tracking/src/trackerKCF.cpp rename to modules/tracking/src/legacy/trackerKCF.legacy.hpp diff --git a/modules/tracking/src/trackerMIL.cpp b/modules/tracking/src/legacy/trackerMIL.legacy.hpp similarity index 100% rename from modules/tracking/src/trackerMIL.cpp rename to modules/tracking/src/legacy/trackerMIL.legacy.hpp From 80c197590ccfbd1dd9b4f54b250d26e06e840683 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 27 Oct 2020 21:18:18 +0000 Subject: [PATCH 2/2] tracking: rework tracking API - simplify Tracker API - fix perf tests (don't measure video decoding) --- modules/tracking/CMakeLists.txt | 23 +- modules/tracking/include/opencv2/tracking.hpp | 271 ++++ .../include/opencv2/tracking/feature.hpp | 13 +- .../opencv2/tracking/kalman_filters.hpp | 19 +- .../opencv2/tracking/onlineBoosting.hpp | 13 +- .../include/opencv2/tracking/onlineMIL.hpp | 9 +- .../include/opencv2/tracking/tldDataset.hpp | 14 +- .../opencv2/tracking/tracking_by_matching.hpp | 11 +- .../opencv2/tracking/tracking_internals.hpp | 799 ++++-------- .../opencv2/tracking/tracking_legacy.hpp | 1118 +---------------- modules/tracking/misc/java/gen_dict.json | 5 + .../java/test/TrackerCreateLegacyTest.java | 23 + .../misc/java/test/TrackerCreateTest.java | 38 + modules/tracking/misc/objc/gen_dict.json | 5 +- .../misc/python/pyopencv_tracking.hpp | 6 + .../misc/python/test/test_tracking_contrib.py | 31 + modules/tracking/perf/perf_Tracker.cpp | 412 ------ modules/tracking/perf/perf_main.cpp | 17 +- modules/tracking/perf/perf_trackers.cpp | 119 ++ modules/tracking/samples/benchmark.cpp | 8 +- modules/tracking/samples/csrt.cpp | 2 +- modules/tracking/samples/goturnTracker.cpp | 11 +- modules/tracking/samples/kcf.cpp | 2 +- .../tracking/samples/multiTracker_dataset.cpp | 4 +- modules/tracking/samples/multitracker.cpp | 6 +- modules/tracking/samples/samples_utility.hpp | 41 +- modules/tracking/samples/tracker.cpp | 8 +- modules/tracking/samples/tracker_dataset.cpp | 8 +- .../tracking/samples/tracking_by_matching.cpp | 5 +- .../tutorial_customizing_cn_tracker.cpp | 2 +- .../tutorial_introduction_to_tracker.cpp | 3 +- .../samples/tutorial_multitracker.cpp | 6 +- modules/tracking/src/PFSolver.hpp | 2 - .../src/augmented_unscented_kalman.cpp | 11 +- modules/tracking/src/feature.cpp | 15 +- modules/tracking/src/featureColorName.cpp | 8 +- modules/tracking/src/gtrTracker.cpp | 144 +-- modules/tracking/src/gtrTracker.hpp | 80 -- modules/tracking/src/gtrUtils.cpp | 1 + modules/tracking/src/gtrUtils.hpp | 1 - modules/tracking/src/kuhn_munkres.cpp | 8 + modules/tracking/src/kuhn_munkres.hpp | 5 + .../tracking/src/legacy/tracker.legacy.hpp | 54 +- .../src/legacy/trackerCSRT.legacy.hpp | 684 +--------- .../tracking/src/legacy/trackerKCF.legacy.hpp | 934 +------------- .../tracking/src/legacy/trackerMIL.legacy.hpp | 264 +--- modules/tracking/src/mosseTracker.cpp | 23 +- modules/tracking/src/multiTracker.cpp | 31 +- modules/tracking/src/multiTracker.hpp | 12 +- modules/tracking/src/multiTracker_alt.cpp | 5 +- modules/tracking/src/onlineBoosting.cpp | 7 +- modules/tracking/src/precomp.hpp | 30 +- modules/tracking/src/tldDataset.cpp | 10 +- modules/tracking/src/tldDetector.cpp | 15 +- modules/tracking/src/tldDetector.hpp | 13 +- .../tracking/src/tldEnsembleClassifier.cpp | 13 +- .../tracking/src/tldEnsembleClassifier.hpp | 14 +- modules/tracking/src/tldModel.cpp | 14 +- modules/tracking/src/tldModel.hpp | 16 +- modules/tracking/src/tldTracker.cpp | 20 +- modules/tracking/src/tldTracker.hpp | 16 +- modules/tracking/src/tldUtils.cpp | 11 +- modules/tracking/src/tldUtils.hpp | 13 +- modules/tracking/src/tracker.cpp | 23 + modules/tracking/src/trackerBoosting.cpp | 10 +- modules/tracking/src/trackerBoostingModel.cpp | 8 +- modules/tracking/src/trackerBoostingModel.hpp | 10 +- modules/tracking/src/trackerCSRT.cpp | 655 ++++++++++ .../src/trackerCSRTScaleEstimation.cpp | 6 +- modules/tracking/src/trackerFeature.cpp | 7 +- modules/tracking/src/trackerFeatureSet.cpp | 8 +- modules/tracking/src/trackerKCF.cpp | 938 ++++++++++++++ modules/tracking/src/trackerMIL.cpp | 265 ++++ modules/tracking/src/trackerMILModel.cpp | 7 +- modules/tracking/src/trackerMILModel.hpp | 9 +- modules/tracking/src/trackerMedianFlow.cpp | 35 +- modules/tracking/src/trackerModel.cpp | 7 +- modules/tracking/src/trackerSampler.cpp | 8 +- .../tracking/src/trackerSamplerAlgorithm.cpp | 17 +- .../tracking/src/trackerStateEstimator.cpp | 7 +- modules/tracking/src/tracking_by_matching.cpp | 10 +- modules/tracking/src/tracking_utils.cpp | 5 +- modules/tracking/src/tracking_utils.hpp | 9 +- modules/tracking/src/unscented_kalman.cpp | 11 +- modules/tracking/test/test_aukf.cpp | 2 +- .../test/test_trackerParametersIO.cpp | 20 +- modules/tracking/test/test_trackers.cpp | 336 +++-- modules/tracking/test/test_ukf.cpp | 2 +- .../tutorials/tutorial_multitracker.markdown | 6 +- 89 files changed, 3621 insertions(+), 4326 deletions(-) create mode 100644 modules/tracking/include/opencv2/tracking.hpp create mode 100644 modules/tracking/misc/java/gen_dict.json create mode 100644 modules/tracking/misc/java/test/TrackerCreateLegacyTest.java create mode 100644 modules/tracking/misc/java/test/TrackerCreateTest.java create mode 100644 modules/tracking/misc/python/pyopencv_tracking.hpp create mode 100644 modules/tracking/misc/python/test/test_tracking_contrib.py delete mode 100644 modules/tracking/perf/perf_Tracker.cpp create mode 100644 modules/tracking/perf/perf_trackers.cpp delete mode 100644 modules/tracking/src/gtrTracker.hpp create mode 100644 modules/tracking/src/tracker.cpp create mode 100644 modules/tracking/src/trackerCSRT.cpp create mode 100644 modules/tracking/src/trackerKCF.cpp create mode 100644 modules/tracking/src/trackerMIL.cpp diff --git a/modules/tracking/CMakeLists.txt b/modules/tracking/CMakeLists.txt index 055a0593e..2834b10ca 100644 --- a/modules/tracking/CMakeLists.txt +++ b/modules/tracking/CMakeLists.txt @@ -1,3 +1,24 @@ set(the_description "Tracking API") -ocv_define_module(tracking opencv_imgproc opencv_core opencv_video opencv_plot OPTIONAL opencv_dnn opencv_datasets WRAP java python objc) + +set(debug_modules "") +if(DEBUG_opencv_tracking) + list(APPEND debug_modules opencv_highgui) +endif() + +ocv_define_module(tracking + opencv_imgproc + opencv_core + opencv_video + opencv_plot # samples only + ${debug_modules} + OPTIONAL + opencv_dnn + opencv_datasets + opencv_highgui + WRAP + java + python + objc +) + ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-shadow /wd4458) diff --git a/modules/tracking/include/opencv2/tracking.hpp b/modules/tracking/include/opencv2/tracking.hpp new file mode 100644 index 000000000..f9d1b154f --- /dev/null +++ b/modules/tracking/include/opencv2/tracking.hpp @@ -0,0 +1,271 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_CONTRIB_TRACKING_HPP +#define OPENCV_CONTRIB_TRACKING_HPP + +#include "opencv2/core.hpp" + +namespace cv { +#ifndef CV_DOXYGEN +inline namespace tracking { +#endif + +/** @defgroup tracking Tracking API +@{ + @defgroup tracking_detail Tracking API implementation details + @defgroup tracking_legacy Legacy Tracking API +@} +*/ + +/** @addtogroup tracking +@{ +Tracking is an important issue for many computer vision applications in real world scenario. +The development in this area is very fragmented and this API is an interface useful for plug several algorithms and compare them. +*/ + + + +/** @brief Base abstract class for the long-term tracker + */ +class CV_EXPORTS_W Tracker +{ +protected: + Tracker(); +public: + virtual ~Tracker(); + + /** @brief Initialize the tracker with a known bounding box that surrounded the target + @param image The initial frame + @param boundingBox The initial bounding box + */ + CV_WRAP virtual + void init(InputArray image, const Rect& boundingBox) = 0; + + /** @brief Update the tracker, find the new most likely bounding box for the target + @param image The current frame + @param boundingBox The bounding box that represent the new target location, if true was returned, not + modified otherwise + + @return True means that target was located and false means that tracker cannot locate target in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ + CV_WRAP virtual + bool update(InputArray image, CV_OUT Rect& boundingBox) = 0; +}; + + +/** @brief the CSRT tracker + +The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability +*/ +class CV_EXPORTS_W TrackerCSRT : public Tracker +{ +protected: + TrackerCSRT(); // use ::create() +public: + virtual ~TrackerCSRT() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + + CV_PROP_RW bool use_hog; + CV_PROP_RW bool use_color_names; + CV_PROP_RW bool use_gray; + CV_PROP_RW bool use_rgb; + CV_PROP_RW bool use_channel_weights; + CV_PROP_RW bool use_segmentation; + + CV_PROP_RW std::string window_function; //!< Window function: "hann", "cheb", "kaiser" + CV_PROP_RW float kaiser_alpha; + CV_PROP_RW float cheb_attenuation; + + CV_PROP_RW float template_size; + CV_PROP_RW float gsl_sigma; + CV_PROP_RW float hog_orientations; + CV_PROP_RW float hog_clip; + CV_PROP_RW float padding; + CV_PROP_RW float filter_lr; + CV_PROP_RW float weights_lr; + CV_PROP_RW int num_hog_channels_used; + CV_PROP_RW int admm_iterations; + CV_PROP_RW int histogram_bins; + CV_PROP_RW float histogram_lr; + CV_PROP_RW int background_ratio; + CV_PROP_RW int number_of_scales; + CV_PROP_RW float scale_sigma_factor; + CV_PROP_RW float scale_model_max_area; + CV_PROP_RW float scale_lr; + CV_PROP_RW float scale_step; + + CV_PROP_RW float psr_threshold; //!< we lost the target, if the psr is lower than this. + }; + + /** @brief Create CSRT tracker instance + @param parameters CSRT parameters TrackerCSRT::Params + */ + static CV_WRAP + Ptr create(const TrackerCSRT::Params ¶meters = TrackerCSRT::Params()); + + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; + + CV_WRAP virtual void setInitialMask(InputArray mask) = 0; +}; + + + +/** @brief the KCF (Kernelized Correlation Filter) tracker + + * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. + * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). + * The original paper of KCF is available at + * as well as the matlab implementation. For more information about KCF with color-names features, please refer to + * . + */ +class CV_EXPORTS_W TrackerKCF : public Tracker +{ +protected: + TrackerKCF(); // use ::create() +public: + virtual ~TrackerKCF() CV_OVERRIDE; + + /** + * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names + * The modes available now: + - "GRAY" -- Use grayscale values as the feature + - "CN" -- Color-names feature + */ + enum MODE { + GRAY = (1 << 0), + CN = (1 << 1), + CUSTOM = (1 << 2) + }; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + + CV_PROP_RW float detect_thresh; //!< detection confidence threshold + CV_PROP_RW float sigma; //!< gaussian kernel bandwidth + CV_PROP_RW float lambda; //!< regularization + CV_PROP_RW float interp_factor; //!< linear interpolation factor for adaptation + CV_PROP_RW float output_sigma_factor; //!< spatial bandwidth (proportional to target) + CV_PROP_RW float pca_learning_rate; //!< compression learning rate + CV_PROP_RW bool resize; //!< activate the resize feature to improve the processing speed + CV_PROP_RW bool split_coeff; //!< split the training coefficients into two matrices + CV_PROP_RW bool wrap_kernel; //!< wrap around the kernel values + CV_PROP_RW bool compress_feature; //!< activate the pca method to compress the features + CV_PROP_RW int max_patch_size; //!< threshold for the ROI size + CV_PROP_RW int compressed_size; //!< feature size after compression + CV_PROP_RW int desc_pca; //!< compressed descriptors of TrackerKCF::MODE + CV_PROP_RW int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE + }; + + /** @brief Create KCF tracker instance + @param parameters KCF parameters TrackerKCF::Params + */ + static CV_WRAP + Ptr create(const TrackerKCF::Params ¶meters = TrackerKCF::Params()); + + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; + + // FIXIT use interface + typedef void (*FeatureExtractorCallbackFN)(const Mat, const Rect, Mat&); + virtual void setFeatureExtractor(FeatureExtractorCallbackFN callback, bool pca_func = false) = 0; +}; + + + +/** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the +background. + +Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is +based on @cite MIL . + +Original code can be found here + */ +class CV_EXPORTS_W TrackerMIL : public Tracker +{ +protected: + TrackerMIL(); // use ::create() +public: + virtual ~TrackerMIL() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + //parameters for sampler + CV_PROP_RW float samplerInitInRadius; //!< radius for gathering positive instances during init + CV_PROP_RW int samplerInitMaxNegNum; //!< # negative samples to use during init + CV_PROP_RW float samplerSearchWinSize; //!< size of search window + CV_PROP_RW float samplerTrackInRadius; //!< radius for gathering positive instances during tracking + CV_PROP_RW int samplerTrackMaxPosNum; //!< # positive samples to use during tracking + CV_PROP_RW int samplerTrackMaxNegNum; //!< # negative samples to use during tracking + CV_PROP_RW int featureSetNumFeatures; //!< # features + }; + + /** @brief Create MIL tracker instance + * @param parameters MIL parameters TrackerMIL::Params + */ + static CV_WRAP + Ptr create(const TrackerMIL::Params ¶meters = TrackerMIL::Params()); + + + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; +}; + + + +/** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker + * + * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, + * GOTURN is much faster due to offline training without online fine-tuning nature. + * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video, + * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly + * robust to viewpoint changes, lighting changes, and deformations. + * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227. + * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2. + * Original paper is here: + * As long as original authors implementation: + * Implementation of training algorithm is placed in separately here due to 3d-party dependencies: + * + * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. + */ +class CV_EXPORTS_W TrackerGOTURN : public Tracker +{ +protected: + TrackerGOTURN(); // use ::create() +public: + virtual ~TrackerGOTURN() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + CV_PROP_RW std::string modelTxt; + CV_PROP_RW std::string modelBin; + }; + + /** @brief Constructor + @param parameters GOTURN parameters TrackerGOTURN::Params + */ + static CV_WRAP + Ptr create(const TrackerGOTURN::Params& parameters = TrackerGOTURN::Params()); + + //void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; +}; + +//! @} + +#ifndef CV_DOXYGEN +} +#endif +} // namespace + +#endif // OPENCV_CONTRIB_TRACKING_HPP diff --git a/modules/tracking/include/opencv2/tracking/feature.hpp b/modules/tracking/include/opencv2/tracking/feature.hpp index 3bcfe6e6c..ebc28ea94 100644 --- a/modules/tracking/include/opencv2/tracking/feature.hpp +++ b/modules/tracking/include/opencv2/tracking/feature.hpp @@ -53,12 +53,15 @@ * TODO Changed CvHaarEvaluator based on ADABOOSTING implementation (Grabner et al.) */ -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { -//! @addtogroup tracking +//! @addtogroup tracking_detail //! @{ +inline namespace feature { + #define FEATURES "features" #define CC_FEATURES FEATURES @@ -409,8 +412,10 @@ inline uchar CvLBPEvaluator::Feature::calc( const Mat &_sum, size_t y ) const ( psum[p[4]] - psum[p[5]] - psum[p[8]] + psum[p[9]] >= cval ? 1 : 0 ) ); // 3 } +} // namespace + //! @} -} /* namespace cv */ +}}} // namespace cv #endif diff --git a/modules/tracking/include/opencv2/tracking/kalman_filters.hpp b/modules/tracking/include/opencv2/tracking/kalman_filters.hpp index 7a89c87dd..0b2e9da05 100644 --- a/modules/tracking/include/opencv2/tracking/kalman_filters.hpp +++ b/modules/tracking/include/opencv2/tracking/kalman_filters.hpp @@ -45,10 +45,14 @@ #include "opencv2/core.hpp" #include -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + +inline namespace kalman_filters { /** @brief The interface for Unscented Kalman filter and Augmented Unscented Kalman filter. */ @@ -222,7 +226,10 @@ CV_EXPORTS Ptr createUnscentedKalmanFilter( const Unscent */ CV_EXPORTS Ptr createAugmentedUnscentedKalmanFilter( const AugmentedUnscentedKalmanFilterParams ¶ms ); -} // tracking -} // cv +} // namespace + +//! @} + +}}} // namespace #endif diff --git a/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp b/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp index 982bc205b..a8638c542 100644 --- a/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp +++ b/modules/tracking/include/opencv2/tracking/onlineBoosting.hpp @@ -44,12 +44,15 @@ #include "opencv2/core.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { -//! @addtogroup tracking +//! @addtogroup tracking_detail //! @{ +inline namespace online_boosting { + //TODO based on the original implementation //http://vision.ucsd.edu/~bbabenko/project_miltrack.shtml @@ -281,8 +284,10 @@ class ClassifierThreshold int m_parity; }; +} // namespace + //! @} -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/include/opencv2/tracking/onlineMIL.hpp b/modules/tracking/include/opencv2/tracking/onlineMIL.hpp index 78e1372f8..9fb341eeb 100644 --- a/modules/tracking/include/opencv2/tracking/onlineMIL.hpp +++ b/modules/tracking/include/opencv2/tracking/onlineMIL.hpp @@ -45,10 +45,11 @@ #include "opencv2/core.hpp" #include -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { -//! @addtogroup tracking +//! @addtogroup tracking_detail //! @{ //TODO based on the original implementation @@ -113,6 +114,6 @@ class ClfOnlineStump //! @} -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index a87425566..1bdd3fb4c 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -44,13 +44,21 @@ #include "opencv2/core.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + namespace tld { CV_EXPORTS cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath = "TLD_dataset", int datasetInd = 0); CV_EXPORTS cv::String tld_getNextDatasetFrame(); } -} + +//! @} + +}}} #endif diff --git a/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp b/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp index b6962e2a9..6590c2a1e 100644 --- a/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp +++ b/modules/tracking/include/opencv2/tracking/tracking_by_matching.hpp @@ -20,6 +20,12 @@ namespace cv { +namespace detail { +inline namespace tracking { + +//! @addtogroup tracking_detail +//! @{ + namespace tbm { //Tracking-by-Matching /// /// \brief The TrackedObject struct defines properties of detected object. @@ -553,5 +559,8 @@ public: CV_EXPORTS cv::Ptr createTrackerByMatching(const TrackerParams ¶ms = TrackerParams()); } // namespace tbm -} // namespace cv + +//! @} + +}}} // namespace #endif // #ifndef __OPENCV_TRACKING_TRACKING_BY_MATCHING_HPP__ diff --git a/modules/tracking/include/opencv2/tracking/tracking_internals.hpp b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp index 3c09c2b77..fee43e3e0 100644 --- a/modules/tracking/include/opencv2/tracking/tracking_internals.hpp +++ b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp @@ -39,30 +39,264 @@ // //M*/ -#ifndef __OPENCV_TRACKER_HPP__ -#define __OPENCV_TRACKER_HPP__ - -#include "opencv2/core.hpp" -#include "opencv2/imgproc/types_c.h" -#include "feature.hpp" -#include "onlineMIL.hpp" -#include "onlineBoosting.hpp" +#ifndef OPENCV_TRACKING_DETAIL_HPP +#define OPENCV_TRACKING_DETAIL_HPP /* * Partially based on: * ==================================================================================================================== - * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation + * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking * * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml * */ -namespace cv -{ +#include "opencv2/core.hpp" + +#include "feature.hpp" // CvHaarEvaluator +#include "onlineBoosting.hpp" // StrongClassifierDirectSelection +#include "onlineMIL.hpp" // ClfMilBoost + +namespace cv { +namespace detail { +inline namespace tracking { + +/** @addtogroup tracking_detail +@{ + +Long-term optical tracking API +------------------------------ + +Long-term optical tracking is an important issue for many computer vision applications in +real world scenario. The development in this area is very fragmented and this API is an unique +interface useful for plug several algorithms and compare them. This work is partially based on +@cite AAM and @cite AMVOT . + +These algorithms start from a bounding box of the target and with their internal representation they +avoid the drift during the tracking. These long-term trackers are able to evaluate online the +quality of the location of the target in the new frame, without ground truth. + +There are three main components: the TrackerSampler, the TrackerFeatureSet and the TrackerModel. The +first component is the object that computes the patches over the frame based on the last target +location. The TrackerFeatureSet is the class that manages the Features, is possible plug many kind +of these (HAAR, HOG, LBP, Feature2D, etc). The last component is the internal representation of the +target, it is the appearance model. It stores all state candidates and compute the trajectory (the +most likely target states). The class TrackerTargetState represents a possible state of the target. +The TrackerSampler and the TrackerFeatureSet are the visual representation of the target, instead +the TrackerModel is the statistical model. + +A recent benchmark between these algorithms can be found in @cite OOT + +Creating Your Own %Tracker +-------------------- + +If you want to create a new tracker, here's what you have to do. First, decide on the name of the class +for the tracker (to meet the existing style, we suggest something with prefix "tracker", e.g. +trackerMIL, trackerBoosting) -- we shall refer to this choice as to "classname" in subsequent. + +- Declare your tracker in modules/tracking/include/opencv2/tracking/tracker.hpp. Your tracker should inherit from + Tracker (please, see the example below). You should declare the specialized Param structure, + where you probably will want to put the data, needed to initialize your tracker. You should + get something similar to : +@code + class CV_EXPORTS_W TrackerMIL : public Tracker + { + public: + struct CV_EXPORTS Params + { + Params(); + //parameters for sampler + float samplerInitInRadius; // radius for gathering positive instances during init + int samplerInitMaxNegNum; // # negative samples to use during init + float samplerSearchWinSize; // size of search window + float samplerTrackInRadius; // radius for gathering positive instances during tracking + int samplerTrackMaxPosNum; // # positive samples to use during tracking + int samplerTrackMaxNegNum; // # negative samples to use during tracking + int featureSetNumFeatures; // #features + + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + }; +@endcode + of course, you can also add any additional methods of your choice. It should be pointed out, + however, that it is not expected to have a constructor declared, as creation should be done via + the corresponding create() method. +- Finally, you should implement the function with signature : +@code + Ptr classname::create(const classname::Params ¶meters){ + ... + } +@endcode + That function can (and probably will) return a pointer to some derived class of "classname", + which will probably have a real constructor. + +Every tracker has three component TrackerSampler, TrackerFeatureSet and TrackerModel. The first two +are instantiated from Tracker base class, instead the last component is abstract, so you must +implement your TrackerModel. + +### TrackerSampler + +TrackerSampler is already instantiated, but you should define the sampling algorithm and add the +classes (or single class) to TrackerSampler. You can choose one of the ready implementation as +TrackerSamplerCSC or you can implement your sampling method, in this case the class must inherit +TrackerSamplerAlgorithm. Fill the samplingImpl method that writes the result in "sample" output +argument. + +Example of creating specialized TrackerSamplerAlgorithm TrackerSamplerCSC : : +@code + class CV_EXPORTS_W TrackerSamplerCSC : public TrackerSamplerAlgorithm + { + public: + TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); + ~TrackerSamplerCSC(); + ... + + protected: + bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ); + ... + + }; +@endcode + +Example of adding TrackerSamplerAlgorithm to TrackerSampler : : +@code + //sampler is the TrackerSampler + Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); + if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) + return false; + + //or add CSC sampler with default parameters + //sampler->addTrackerSamplerAlgorithm( "CSC" ); +@endcode +@sa + TrackerSamplerCSC, TrackerSamplerAlgorithm + +### TrackerFeatureSet + +TrackerFeatureSet is already instantiated (as first) , but you should define what kinds of features +you'll use in your tracker. You can use multiple feature types, so you can add a ready +implementation as TrackerFeatureHAAR in your TrackerFeatureSet or develop your own implementation. +In this case, in the computeImpl method put the code that extract the features and in the selection +method optionally put the code for the refinement and selection of the features. + +Example of creating specialized TrackerFeature TrackerFeatureHAAR : : +@code + class CV_EXPORTS_W TrackerFeatureHAAR : public TrackerFeature + { + public: + TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); + ~TrackerFeatureHAAR(); + void selection( Mat& response, int npoints ); + ... + + protected: + bool computeImpl( const std::vector& images, Mat& response ); + ... + + }; +@endcode +Example of adding TrackerFeature to TrackerFeatureSet : : +@code + //featureSet is the TrackerFeatureSet + Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); + featureSet->addTrackerFeature( trackerFeature ); +@endcode +@sa + TrackerFeatureHAAR, TrackerFeatureSet + +### TrackerModel + +TrackerModel is abstract, so in your implementation you must develop your TrackerModel that inherit +from TrackerModel. Fill the method for the estimation of the state "modelEstimationImpl", that +estimates the most likely target location, see @cite AAM table I (ME) for further information. Fill +"modelUpdateImpl" in order to update the model, see @cite AAM table I (MU). In this class you can use +the :cConfidenceMap and :cTrajectory to storing the model. The first represents the model on the all +possible candidate states and the second represents the list of all estimated states. + +Example of creating specialized TrackerModel TrackerMILModel : : +@code + class TrackerMILModel : public TrackerModel + { + public: + TrackerMILModel( const Rect& boundingBox ); + ~TrackerMILModel(); + ... + + protected: + void modelEstimationImpl( const std::vector& responses ); + void modelUpdateImpl(); + ... + + }; +@endcode +And add it in your Tracker : : +@code + bool TrackerMIL::initImpl( const Mat& image, const Rect2d& boundingBox ) + { + ... + //model is the general TrackerModel field of the general Tracker + model = new TrackerMILModel( boundingBox ); + ... + } +@endcode +In the last step you should define the TrackerStateEstimator based on your implementation or you can +use one of ready class as TrackerStateEstimatorMILBoosting. It represent the statistical part of the +model that estimates the most likely target state. + +Example of creating specialized TrackerStateEstimator TrackerStateEstimatorMILBoosting : : +@code + class CV_EXPORTS_W TrackerStateEstimatorMILBoosting : public TrackerStateEstimator + { + class TrackerMILTargetState : public TrackerTargetState + { + ... + }; + + public: + TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); + ~TrackerStateEstimatorMILBoosting(); + ... + + protected: + Ptr estimateImpl( const std::vector& confidenceMaps ); + void updateImpl( std::vector& confidenceMaps ); + ... + + }; +@endcode +And add it in your TrackerModel : : +@code + //model is the TrackerModel of your Tracker + Ptr stateEstimator = new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ); + model->setTrackerStateEstimator( stateEstimator ); +@endcode +@sa + TrackerModel, TrackerStateEstimatorMILBoosting, TrackerTargetState + +During this step, you should define your TrackerTargetState based on your implementation. +TrackerTargetState base class has only the bounding box (upper-left position, width and height), you +can enrich it adding scale factor, target rotation, etc. + +Example of creating specialized TrackerTargetState TrackerMILTargetState : : +@code + class TrackerMILTargetState : public TrackerTargetState + { + public: + TrackerMILTargetState( const Point2f& position, int targetWidth, int targetHeight, bool foreground, const Mat& features ); + ~TrackerMILTargetState(); + ... + + private: + bool isTarget; + Mat targetFeatures; + ... + + }; +@endcode + +*/ -//! @addtogroup tracking -//! @{ /************************************ TrackerFeature Base Classes ************************************/ @@ -517,50 +751,6 @@ class CV_EXPORTS TrackerModel }; -/************************************ Tracker Base Class ************************************/ - -/** @brief Base abstract class for the long-term tracker: - */ -class CV_EXPORTS_W Tracker : public virtual Algorithm -{ - public: - - virtual ~Tracker() CV_OVERRIDE; - - /** @brief Initialize the tracker with a known bounding box that surrounded the target - @param image The initial frame - @param boundingBox The initial bounding box - - @return True if initialization went succesfully, false otherwise - */ - CV_WRAP bool init( InputArray image, const Rect2d& boundingBox ); - - /** @brief Update the tracker, find the new most likely bounding box for the target - @param image The current frame - @param boundingBox The bounding box that represent the new target location, if true was returned, not - modified otherwise - - @return True means that target was located and false means that tracker cannot locate target in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - CV_WRAP bool update( InputArray image, CV_OUT Rect2d& boundingBox ); - - virtual void read( const FileNode& fn ) CV_OVERRIDE = 0; - virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0; - - protected: - - virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; - virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; - - bool isInit; - - Ptr featureSet; - Ptr sampler; - Ptr model; -}; - /************************************ Specific TrackerStateEstimator Classes ************************************/ @@ -1052,497 +1242,8 @@ class CV_EXPORTS TrackerFeatureLBP : public TrackerFeature }; -/************************************ Specific Tracker Classes ************************************/ - -/** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the -background. - -Multiple Instance Learning avoids the drift problem for a robust tracking. The implementation is -based on @cite MIL . - -Original code can be found here - */ -class CV_EXPORTS_W TrackerMIL : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - //parameters for sampler - float samplerInitInRadius; //!< radius for gathering positive instances during init - int samplerInitMaxNegNum; //!< # negative samples to use during init - float samplerSearchWinSize; //!< size of search window - float samplerTrackInRadius; //!< radius for gathering positive instances during tracking - int samplerTrackMaxPosNum; //!< # positive samples to use during tracking - int samplerTrackMaxNegNum; //!< # negative samples to use during tracking - int featureSetNumFeatures; //!< # features - - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; - - /** @brief Constructor - @param parameters MIL parameters TrackerMIL::Params - */ - static Ptr create(const TrackerMIL::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerMIL() CV_OVERRIDE {} -}; - -/** @brief the Boosting tracker - -This is a real-time object tracking based on a novel on-line version of the AdaBoost algorithm. -The classifier uses the surrounding background as negative examples in update step to avoid the -drifting problem. The implementation is based on @cite OLB . - */ -class CV_EXPORTS_W TrackerBoosting : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - int numClassifiers; //! create(const TrackerBoosting::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerBoosting() CV_OVERRIDE {} -}; - -/** @brief the Median Flow tracker - -Implementation of a paper @cite MedianFlow . - -The tracker is suitable for very smooth and predictable movements when object is visible throughout -the whole sequence. It's quite and accurate for this type of problems (in particular, it was shown -by authors to outperform MIL). During the implementation period the code at -, the courtesy of the author Arthur Amarra, was used for the -reference purpose. - */ -class CV_EXPORTS_W TrackerMedianFlow : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); //! create(const TrackerMedianFlow::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerMedianFlow() CV_OVERRIDE {} -}; - -/** @brief the TLD (Tracking, learning and detection) tracker - -TLD is a novel tracking framework that explicitly decomposes the long-term tracking task into -tracking, learning and detection. - -The tracker follows the object from frame to frame. The detector localizes all appearances that -have been observed so far and corrects the tracker if necessary. The learning estimates detector's -errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . - -The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this -implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial -occlusions, object absence etc. - */ -class CV_EXPORTS_W TrackerTLD : public Tracker -{ - public: - struct CV_EXPORTS Params - { - Params(); - void read( const FileNode& /*fn*/ ); - void write( FileStorage& /*fs*/ ) const; - }; - - /** @brief Constructor - @param parameters TLD parameters TrackerTLD::Params - */ - static Ptr create(const TrackerTLD::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerTLD() CV_OVERRIDE {} -}; - -/** @brief the KCF (Kernelized Correlation Filter) tracker - - * KCF is a novel tracking framework that utilizes properties of circulant matrix to enhance the processing speed. - * This tracking method is an implementation of @cite KCF_ECCV which is extended to KCF with color-names features (@cite KCF_CN). - * The original paper of KCF is available at - * as well as the matlab implementation. For more information about KCF with color-names features, please refer to - * . - */ -class CV_EXPORTS_W TrackerKCF : public Tracker -{ -public: - /** - * \brief Feature type to be used in the tracking grayscale, colornames, compressed color-names - * The modes available now: - - "GRAY" -- Use grayscale values as the feature - - "CN" -- Color-names feature - */ - enum MODE { - GRAY = (1 << 0), - CN = (1 << 1), - CUSTOM = (1 << 2) - }; - - struct CV_EXPORTS Params - { - /** - * \brief Constructor - */ - Params(); - - /** - * \brief Read parameters from a file - */ - void read(const FileNode& /*fn*/); - - /** - * \brief Write parameters to a file - */ - void write(FileStorage& /*fs*/) const; - - float detect_thresh; //!< detection confidence threshold - float sigma; //!< gaussian kernel bandwidth - float lambda; //!< regularization - float interp_factor; //!< linear interpolation factor for adaptation - float output_sigma_factor; //!< spatial bandwidth (proportional to target) - float pca_learning_rate; //!< compression learning rate - bool resize; //!< activate the resize feature to improve the processing speed - bool split_coeff; //!< split the training coefficients into two matrices - bool wrap_kernel; //!< wrap around the kernel values - bool compress_feature; //!< activate the pca method to compress the features - int max_patch_size; //!< threshold for the ROI size - int compressed_size; //!< feature size after compression - int desc_pca; //!< compressed descriptors of TrackerKCF::MODE - int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE - }; - - virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0; - - /** @brief Constructor - @param parameters KCF parameters TrackerKCF::Params - */ - static Ptr create(const TrackerKCF::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerKCF() CV_OVERRIDE {} -}; - -/** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker - - * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, - * GOTURN is much faster due to offline training without online fine-tuning nature. - * GOTURN tracker addresses the problem of single target tracking: given a bounding box label of an object in the first frame of the video, - * we track that object through the rest of the video. NOTE: Current method of GOTURN does not handle occlusions; however, it is fairly - * robust to viewpoint changes, lighting changes, and deformations. - * Inputs of GOTURN are two RGB patches representing Target and Search patches resized to 227x227. - * Outputs of GOTURN are predicted bounding box coordinates, relative to Search patch coordinate system, in format X1,Y1,X2,Y2. - * Original paper is here: - * As long as original authors implementation: - * Implementation of training algorithm is placed in separately here due to 3d-party dependencies: - * - * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. -*/ -class CV_EXPORTS_W TrackerGOTURN : public Tracker -{ -public: - struct CV_EXPORTS Params - { - Params(); - void read(const FileNode& /*fn*/); - void write(FileStorage& /*fs*/) const; - String modelTxt; - String modelBin; - }; - - /** @brief Constructor - @param parameters GOTURN parameters TrackerGOTURN::Params - */ - static Ptr create(const TrackerGOTURN::Params ¶meters); - - CV_WRAP static Ptr create(); - - virtual ~TrackerGOTURN() CV_OVERRIDE {} -}; - -/** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker - -The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive Correlation Filters -@note this tracker works with grayscale images, if passed bgr ones, they will get converted internally. -*/ - -class CV_EXPORTS_W TrackerMOSSE : public Tracker -{ - public: - /** @brief Constructor - */ - CV_WRAP static Ptr create(); - - virtual ~TrackerMOSSE() CV_OVERRIDE {} -}; - - -/************************************ MultiTracker Class ---By Laksono Kurnianggoro---) ************************************/ -/** @brief This class is used to track multiple objects using the specified tracker algorithm. - -* The %MultiTracker is naive implementation of multiple object tracking. -* It process the tracked objects independently without any optimization accross the tracked objects. -*/ -class CV_EXPORTS_W MultiTracker : public Algorithm -{ -public: - - /** - * \brief Constructor. - */ - CV_WRAP MultiTracker(); - - /** - * \brief Destructor - */ - ~MultiTracker() CV_OVERRIDE; - - /** - * \brief Add a new object to be tracked. - * - * @param newTracker tracking algorithm to be used - * @param image input image - * @param boundingBox a rectangle represents ROI of the tracked object - */ - CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); - - /** - * \brief Add a set of objects to be tracked. - * @param newTrackers list of tracking algorithms to be used - * @param image input image - * @param boundingBox list of the tracked objects - */ - bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); - - /** - * \brief Update the current tracking status. - * The result will be saved in the internal storage. - * @param image input image - */ - bool update(InputArray image); - - /** - * \brief Update the current tracking status. - * @param image input image - * @param boundingBox the tracking result, represent a list of ROIs of the tracked objects. - */ - CV_WRAP bool update(InputArray image, CV_OUT std::vector & boundingBox); - - /** - * \brief Returns a reference to a storage for the tracked objects, each object corresponds to one tracker algorithm - */ - CV_WRAP const std::vector& getObjects() const; - - /** - * \brief Returns a pointer to a new instance of MultiTracker - */ - CV_WRAP static Ptr create(); - -protected: - //!< storage for the tracker algorithms. - std::vector< Ptr > trackerList; - - //!< storage for the tracked objects, each object corresponds to one tracker algorithm. - std::vector objects; -}; - -/************************************ Multi-Tracker Classes ---By Tyan Vladimir---************************************/ - -/** @brief Base abstract class for the long-term Multi Object Trackers: - -@sa Tracker, MultiTrackerTLD -*/ -class CV_EXPORTS MultiTracker_Alt -{ -public: - /** @brief Constructor for Multitracker - */ - MultiTracker_Alt() - { - targetNum = 0; - } - - /** @brief Add a new target to a tracking-list and initialize the tracker with a known bounding box that surrounded the target - @param image The initial frame - @param boundingBox The initial bounding box of target - @param tracker_algorithm Multi-tracker algorithm - - @return True if new target initialization went succesfully, false otherwise - */ - bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); - - /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets - @param image The current frame - - @return True means that all targets were located and false means that tracker couldn't locate one of the targets in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - bool update(InputArray image); - - /** @brief Current number of targets in tracking-list - */ - int targetNum; - - /** @brief Trackers list for Multi-Object-Tracker - */ - std::vector > trackers; - - /** @brief Bounding Boxes list for Multi-Object-Tracker - */ - std::vector boundingBoxes; - /** @brief List of randomly generated colors for bounding boxes display - */ - std::vector colors; -}; - -/** @brief Multi Object %Tracker for TLD. - -TLD is a novel tracking framework that explicitly decomposes -the long-term tracking task into tracking, learning and detection. - -The tracker follows the object from frame to frame. The detector localizes all appearances that -have been observed so far and corrects the tracker if necessary. The learning estimates detector's -errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . - -The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this -implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial -occlusions, object absence etc. - -@sa Tracker, MultiTracker, TrackerTLD -*/ -class CV_EXPORTS MultiTrackerTLD : public MultiTracker_Alt -{ -public: - /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by - optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation - is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20% - - @param image The current frame. - - @return True means that all targets were located and false means that tracker couldn't locate one of the targets in - current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed - missing from the frame (say, out of sight) - */ - bool update_opt(InputArray image); -}; - -/*********************************** CSRT ************************************/ -/** @brief the CSRT tracker - -The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability -*/ -class CV_EXPORTS_W TrackerCSRT : public Tracker -{ -public: - struct CV_EXPORTS Params - { - /** - * \brief Constructor - */ - Params(); - - /** - * \brief Read parameters from a file - */ - void read(const FileNode& /*fn*/); - - /** - * \brief Write parameters to a file - */ - void write(cv::FileStorage& fs) const; - - bool use_hog; - bool use_color_names; - bool use_gray; - bool use_rgb; - bool use_channel_weights; - bool use_segmentation; - - std::string window_function; //!< Window function: "hann", "cheb", "kaiser" - float kaiser_alpha; - float cheb_attenuation; - - float template_size; - float gsl_sigma; - float hog_orientations; - float hog_clip; - float padding; - float filter_lr; - float weights_lr; - int num_hog_channels_used; - int admm_iterations; - int histogram_bins; - float histogram_lr; - int background_ratio; - int number_of_scales; - float scale_sigma_factor; - float scale_model_max_area; - float scale_lr; - float scale_step; - - float psr_threshold; //!< we lost the target, if the psr is lower than this. - }; - - /** @brief Constructor - @param parameters CSRT parameters TrackerCSRT::Params - */ - static Ptr create(const TrackerCSRT::Params ¶meters); - - CV_WRAP static Ptr create(); - - CV_WRAP virtual void setInitialMask(InputArray mask) = 0; - - virtual ~TrackerCSRT() CV_OVERRIDE {} -}; - //! @} -} /* namespace cv */ -#endif +}}} // namespace + +#endif // OPENCV_TRACKING_DETAIL_HPP diff --git a/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp index 3c09c2b77..bf0bef33c 100644 --- a/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp +++ b/modules/tracking/include/opencv2/tracking/tracking_legacy.hpp @@ -39,483 +39,31 @@ // //M*/ -#ifndef __OPENCV_TRACKER_HPP__ -#define __OPENCV_TRACKER_HPP__ - -#include "opencv2/core.hpp" -#include "opencv2/imgproc/types_c.h" -#include "feature.hpp" -#include "onlineMIL.hpp" -#include "onlineBoosting.hpp" +#ifndef OPENCV_TRACKING_LEGACY_HPP +#define OPENCV_TRACKING_LEGACY_HPP /* * Partially based on: * ==================================================================================================================== - * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation + * - [AAM] S. Salti, A. Cavallaro, L. Di Stefano, Adaptive Appearance Modeling for Video Tracking: Survey and Evaluation * - [AMVOT] X. Li, W. Hu, C. Shen, Z. Zhang, A. Dick, A. van den Hengel, A Survey of Appearance Models in Visual Object Tracking * * This Tracking API has been designed with PlantUML. If you modify this API please change UML files under modules/tracking/doc/uml * */ -namespace cv -{ - -//! @addtogroup tracking -//! @{ - -/************************************ TrackerFeature Base Classes ************************************/ - -/** @brief Abstract base class for TrackerFeature that represents the feature. - */ -class CV_EXPORTS TrackerFeature -{ - public: - virtual ~TrackerFeature(); - - /** @brief Compute the features in the images collection - @param images The images - @param response The output response - */ - void compute( const std::vector& images, Mat& response ); - - /** @brief Create TrackerFeature by tracker feature type - @param trackerFeatureType The TrackerFeature name - - The modes available now: - - - "HAAR" -- Haar Feature-based - - The modes that will be available soon: - - - "HOG" -- Histogram of Oriented Gradients features - - "LBP" -- Local Binary Pattern features - - "FEATURE2D" -- All types of Feature2D - */ - static Ptr create( const String& trackerFeatureType ); - - /** @brief Identify most effective features - @param response Collection of response for the specific TrackerFeature - @param npoints Max number of features - - @note This method modifies the response parameter - */ - virtual void selection( Mat& response, int npoints ) = 0; - - /** @brief Get the name of the specific TrackerFeature - */ - String getClassName() const; - - protected: - - virtual bool computeImpl( const std::vector& images, Mat& response ) = 0; - - String className; -}; - -/** @brief Class that manages the extraction and selection of features - -@cite AAM Feature Extraction and Feature Set Refinement (Feature Processing and Feature Selection). -See table I and section III C @cite AMVOT Appearance modelling -\> Visual representation (Table II, -section 3.1 - 3.2) - -TrackerFeatureSet is an aggregation of TrackerFeature - -@sa - TrackerFeature - - */ -class CV_EXPORTS TrackerFeatureSet -{ - public: - - TrackerFeatureSet(); - - ~TrackerFeatureSet(); - - /** @brief Extract features from the images collection - @param images The input images - */ - void extraction( const std::vector& images ); - - /** @brief Identify most effective features for all feature types (optional) - */ - void selection(); - - /** @brief Remove outliers for all feature types (optional) - */ - void removeOutliers(); - - /** @brief Add TrackerFeature in the collection. Return true if TrackerFeature is added, false otherwise - @param trackerFeatureType The TrackerFeature name - - The modes available now: - - - "HAAR" -- Haar Feature-based - - The modes that will be available soon: - - - "HOG" -- Histogram of Oriented Gradients features - - "LBP" -- Local Binary Pattern features - - "FEATURE2D" -- All types of Feature2D - - Example TrackerFeatureSet::addTrackerFeature : : - @code - //sample usage: - - Ptr trackerFeature = new TrackerFeatureHAAR( HAARparameters ); - featureSet->addTrackerFeature( trackerFeature ); - - //or add CSC sampler with default parameters - //featureSet->addTrackerFeature( "HAAR" ); - @endcode - @note If you use the second method, you must initialize the TrackerFeature - */ - bool addTrackerFeature( String trackerFeatureType ); - - /** @overload - @param feature The TrackerFeature class - */ - bool addTrackerFeature( Ptr& feature ); - - /** @brief Get the TrackerFeature collection (TrackerFeature name, TrackerFeature pointer) - */ - const std::vector > >& getTrackerFeature() const; - - /** @brief Get the responses - - @note Be sure to call extraction before getResponses Example TrackerFeatureSet::getResponses : : - */ - const std::vector& getResponses() const; - - private: - - void clearResponses(); - bool blockAddTrackerFeature; - - std::vector > > features; //list of features - std::vector responses; //list of response after compute - -}; - -/************************************ TrackerSampler Base Classes ************************************/ - -/** @brief Abstract base class for TrackerSamplerAlgorithm that represents the algorithm for the specific -sampler. - */ -class CV_EXPORTS TrackerSamplerAlgorithm -{ - public: - /** - * \brief Destructor - */ - virtual ~TrackerSamplerAlgorithm(); - - /** @brief Create TrackerSamplerAlgorithm by tracker sampler type. - @param trackerSamplerType The trackerSamplerType name - - The modes available now: - - - "CSC" -- Current State Center - - "CS" -- Current State - */ - static Ptr create( const String& trackerSamplerType ); - - /** @brief Computes the regions starting from a position in an image. - - Return true if samples are computed, false otherwise - - @param image The current frame - @param boundingBox The bounding box from which regions can be calculated - - @param sample The computed samples @cite AAM Fig. 1 variable Sk - */ - bool sampling( const Mat& image, Rect boundingBox, std::vector& sample ); - - /** @brief Get the name of the specific TrackerSamplerAlgorithm - */ - String getClassName() const; - - protected: - String className; - - virtual bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) = 0; -}; - -/** - * \brief Class that manages the sampler in order to select regions for the update the model of the tracker - * [AAM] Sampling e Labeling. See table I and section III B - */ - -/** @brief Class that manages the sampler in order to select regions for the update the model of the tracker - -@cite AAM Sampling e Labeling. See table I and section III B - -TrackerSampler is an aggregation of TrackerSamplerAlgorithm -@sa - TrackerSamplerAlgorithm - */ -class CV_EXPORTS TrackerSampler -{ - public: - - /** - * \brief Constructor - */ - TrackerSampler(); - - /** - * \brief Destructor - */ - ~TrackerSampler(); - - /** @brief Computes the regions starting from a position in an image - @param image The current frame - @param boundingBox The bounding box from which regions can be calculated - */ - void sampling( const Mat& image, Rect boundingBox ); - - /** @brief Return the collection of the TrackerSamplerAlgorithm - */ - const std::vector > >& getSamplers() const; - - /** @brief Return the samples from all TrackerSamplerAlgorithm, @cite AAM Fig. 1 variable Sk - */ - const std::vector& getSamples() const; - - /** @brief Add TrackerSamplerAlgorithm in the collection. Return true if sampler is added, false otherwise - @param trackerSamplerAlgorithmType The TrackerSamplerAlgorithm name - - The modes available now: - - "CSC" -- Current State Center - - "CS" -- Current State - - "PF" -- Particle Filtering - - Example TrackerSamplerAlgorithm::addTrackerSamplerAlgorithm : : - @code - TrackerSamplerCSC::Params CSCparameters; - Ptr CSCSampler = new TrackerSamplerCSC( CSCparameters ); - - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; - - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); - @endcode - @note If you use the second method, you must initialize the TrackerSamplerAlgorithm - */ - bool addTrackerSamplerAlgorithm( String trackerSamplerAlgorithmType ); - - /** @overload - @param sampler The TrackerSamplerAlgorithm - */ - bool addTrackerSamplerAlgorithm( Ptr& sampler ); - - private: - std::vector > > samplers; - std::vector samples; - bool blockAddTrackerSampler; - - void clearSamples(); -}; - -/************************************ TrackerModel Base Classes ************************************/ - -/** @brief Abstract base class for TrackerTargetState that represents a possible state of the target. - -See @cite AAM \f$\hat{x}^{i}_{k}\f$ all the states candidates. - -Inherits this class with your Target state, In own implementation you can add scale variation, -width, height, orientation, etc. - */ -class CV_EXPORTS TrackerTargetState -{ - public: - virtual ~TrackerTargetState() - { - } - ; - /** - * \brief Get the position - * \return The position - */ - Point2f getTargetPosition() const; - - /** - * \brief Set the position - * \param position The position - */ - void setTargetPosition( const Point2f& position ); - /** - * \brief Get the width of the target - * \return The width of the target - */ - int getTargetWidth() const; - - /** - * \brief Set the width of the target - * \param width The width of the target - */ - void setTargetWidth( int width ); - /** - * \brief Get the height of the target - * \return The height of the target - */ - int getTargetHeight() const; - - /** - * \brief Set the height of the target - * \param height The height of the target - */ - void setTargetHeight( int height ); - - protected: - Point2f targetPosition; - int targetWidth; - int targetHeight; - -}; - -/** @brief Represents the model of the target at frame \f$k\f$ (all states and scores) - -See @cite AAM The set of the pair \f$\langle \hat{x}^{i}_{k}, C^{i}_{k} \rangle\f$ -@sa TrackerTargetState - */ -typedef std::vector, float> > ConfidenceMap; - -/** @brief Represents the estimate states for all frames - -@cite AAM \f$x_{k}\f$ is the trajectory of the target up to time \f$k\f$ - -@sa TrackerTargetState - */ -typedef std::vector > Trajectory; - -/** @brief Abstract base class for TrackerStateEstimator that estimates the most likely target state. - -See @cite AAM State estimator - -See @cite AMVOT Statistical modeling (Fig. 3), Table III (generative) - IV (discriminative) - V (hybrid) - */ -class CV_EXPORTS TrackerStateEstimator -{ - public: - virtual ~TrackerStateEstimator(); - - /** @brief Estimate the most likely target state, return the estimated state - @param confidenceMaps The overall appearance model as a list of :cConfidenceMap - */ - Ptr estimate( const std::vector& confidenceMaps ); - - /** @brief Update the ConfidenceMap with the scores - @param confidenceMaps The overall appearance model as a list of :cConfidenceMap - */ - void update( std::vector& confidenceMaps ); - - /** @brief Create TrackerStateEstimator by tracker state estimator type - @param trackeStateEstimatorType The TrackerStateEstimator name - - The modes available now: - - - "BOOSTING" -- Boosting-based discriminative appearance models. See @cite AMVOT section 4.4 - - The modes available soon: - - - "SVM" -- SVM-based discriminative appearance models. See @cite AMVOT section 4.5 - */ - static Ptr create( const String& trackeStateEstimatorType ); - - /** @brief Get the name of the specific TrackerStateEstimator - */ - String getClassName() const; - - protected: - - virtual Ptr estimateImpl( const std::vector& confidenceMaps ) = 0; - virtual void updateImpl( std::vector& confidenceMaps ) = 0; - String className; -}; - -/** @brief Abstract class that represents the model of the target. It must be instantiated by specialized -tracker +#include "tracking_internals.hpp" -See @cite AAM Ak - -Inherits this with your TrackerModel - */ -class CV_EXPORTS TrackerModel -{ - public: - - /** - * \brief Constructor - */ - TrackerModel(); - - /** - * \brief Destructor - */ - virtual ~TrackerModel(); - - /** @brief Set TrackerEstimator, return true if the tracker state estimator is added, false otherwise - @param trackerStateEstimator The TrackerStateEstimator - @note You can add only one TrackerStateEstimator - */ - bool setTrackerStateEstimator( Ptr trackerStateEstimator ); - - /** @brief Estimate the most likely target location - - @cite AAM ME, Model Estimation table I - @param responses Features extracted from TrackerFeatureSet - */ - void modelEstimation( const std::vector& responses ); - - /** @brief Update the model - - @cite AAM MU, Model Update table I - */ - void modelUpdate(); - - /** @brief Run the TrackerStateEstimator, return true if is possible to estimate a new state, false otherwise - */ - bool runStateEstimator(); - - /** @brief Set the current TrackerTargetState in the Trajectory - @param lastTargetState The current TrackerTargetState - */ - void setLastTargetState( const Ptr& lastTargetState ); - - /** @brief Get the last TrackerTargetState from Trajectory - */ - Ptr getLastTargetState() const; - - /** @brief Get the list of the ConfidenceMap - */ - const std::vector& getConfidenceMaps() const; - - /** @brief Get the last ConfidenceMap for the current frame - */ - const ConfidenceMap& getLastConfidenceMap() const; - - /** @brief Get the TrackerStateEstimator - */ - Ptr getTrackerStateEstimator() const; - - private: - - void clearCurrentConfidenceMap(); - - protected: - std::vector confidenceMaps; - Ptr stateEstimator; - ConfidenceMap currentConfidenceMap; - Trajectory trajectory; - int maxCMLength; - - virtual void modelEstimationImpl( const std::vector& responses ) = 0; - virtual void modelUpdateImpl() = 0; +namespace cv { +namespace legacy { +#ifndef CV_DOXYGEN +inline namespace tracking { +#endif +using namespace cv::detail::tracking; -}; +/** @addtogroup tracking_legacy +@{ +*/ /************************************ Tracker Base Class ************************************/ @@ -524,7 +72,7 @@ class CV_EXPORTS TrackerModel class CV_EXPORTS_W Tracker : public virtual Algorithm { public: - + Tracker(); virtual ~Tracker() CV_OVERRIDE; /** @brief Initialize the tracker with a known bounding box that surrounded the target @@ -562,496 +110,6 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm }; -/************************************ Specific TrackerStateEstimator Classes ************************************/ - -/** @brief TrackerStateEstimator based on Boosting - */ -class CV_EXPORTS TrackerStateEstimatorMILBoosting : public TrackerStateEstimator -{ - public: - - /** - * Implementation of the target state for TrackerStateEstimatorMILBoosting - */ - class TrackerMILTargetState : public TrackerTargetState - { - - public: - /** - * \brief Constructor - * \param position Top left corner of the bounding box - * \param width Width of the bounding box - * \param height Height of the bounding box - * \param foreground label for target or background - * \param features features extracted - */ - TrackerMILTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& features ); - - /** - * \brief Destructor - */ - ~TrackerMILTargetState() - { - } - ; - - /** @brief Set label: true for target foreground, false for background - @param foreground Label for background/foreground - */ - void setTargetFg( bool foreground ); - /** @brief Set the features extracted from TrackerFeatureSet - @param features The features extracted - */ - void setFeatures( const Mat& features ); - /** @brief Get the label. Return true for target foreground, false for background - */ - bool isTargetFg() const; - /** @brief Get the features extracted - */ - Mat getFeatures() const; - - private: - bool isTarget; - Mat targetFeatures; - }; - - /** @brief Constructor - @param nFeatures Number of features for each sample - */ - TrackerStateEstimatorMILBoosting( int nFeatures = 250 ); - ~TrackerStateEstimatorMILBoosting(); - - /** @brief Set the current confidenceMap - @param confidenceMap The current :cConfidenceMap - */ - void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; - - private: - uint max_idx( const std::vector &v ); - void prepareData( const ConfidenceMap& confidenceMap, Mat& positive, Mat& negative ); - - ClfMilBoost boostMILModel; - bool trained; - int numFeatures; - - ConfidenceMap currentConfidenceMap; -}; - -/** @brief TrackerStateEstimatorAdaBoosting based on ADA-Boosting - */ -class CV_EXPORTS TrackerStateEstimatorAdaBoosting : public TrackerStateEstimator -{ - public: - /** @brief Implementation of the target state for TrackerAdaBoostingTargetState - */ - class TrackerAdaBoostingTargetState : public TrackerTargetState - { - - public: - /** - * \brief Constructor - * \param position Top left corner of the bounding box - * \param width Width of the bounding box - * \param height Height of the bounding box - * \param foreground label for target or background - * \param responses list of features - */ - TrackerAdaBoostingTargetState( const Point2f& position, int width, int height, bool foreground, const Mat& responses ); - - /** - * \brief Destructor - */ - ~TrackerAdaBoostingTargetState() - { - } - ; - - /** @brief Set the features extracted from TrackerFeatureSet - @param responses The features extracted - */ - void setTargetResponses( const Mat& responses ); - /** @brief Set label: true for target foreground, false for background - @param foreground Label for background/foreground - */ - void setTargetFg( bool foreground ); - /** @brief Get the features extracted - */ - Mat getTargetResponses() const; - /** @brief Get the label. Return true for target foreground, false for background - */ - bool isTargetFg() const; - - private: - bool isTarget; - Mat targetResponses; - - }; - - /** @brief Constructor - @param numClassifer Number of base classifiers - @param initIterations Number of iterations in the initialization - @param nFeatures Number of features/weak classifiers - @param patchSize tracking rect - @param ROI initial ROI - */ - TrackerStateEstimatorAdaBoosting( int numClassifer, int initIterations, int nFeatures, Size patchSize, const Rect& ROI ); - - /** - * \brief Destructor - */ - ~TrackerStateEstimatorAdaBoosting(); - - /** @brief Get the sampling ROI - */ - Rect getSampleROI() const; - - /** @brief Set the sampling ROI - @param ROI the sampling ROI - */ - void setSampleROI( const Rect& ROI ); - - /** @brief Set the current confidenceMap - @param confidenceMap The current :cConfidenceMap - */ - void setCurrentConfidenceMap( ConfidenceMap& confidenceMap ); - - /** @brief Get the list of the selected weak classifiers for the classification step - */ - std::vector computeSelectedWeakClassifier(); - - /** @brief Get the list of the weak classifiers that should be replaced - */ - std::vector computeReplacedClassifier(); - - /** @brief Get the list of the weak classifiers that replace those to be replaced - */ - std::vector computeSwappedClassifier(); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; - - Ptr boostClassifier; - - private: - int numBaseClassifier; - int iterationInit; - int numFeatures; - bool trained; - Size initPatchSize; - Rect sampleROI; - std::vector replacedClassifier; - std::vector swappedClassifier; - - ConfidenceMap currentConfidenceMap; -}; - -/** - * \brief TrackerStateEstimator based on SVM - */ -class CV_EXPORTS TrackerStateEstimatorSVM : public TrackerStateEstimator -{ - public: - TrackerStateEstimatorSVM(); - ~TrackerStateEstimatorSVM(); - - protected: - Ptr estimateImpl( const std::vector& confidenceMaps ) CV_OVERRIDE; - void updateImpl( std::vector& confidenceMaps ) CV_OVERRIDE; -}; - -/************************************ Specific TrackerSamplerAlgorithm Classes ************************************/ - -/** @brief TrackerSampler based on CSC (current state centered), used by MIL algorithm TrackerMIL - */ -class CV_EXPORTS TrackerSamplerCSC : public TrackerSamplerAlgorithm -{ - public: - enum - { - MODE_INIT_POS = 1, //!< mode for init positive samples - MODE_INIT_NEG = 2, //!< mode for init negative samples - MODE_TRACK_POS = 3, //!< mode for update positive samples - MODE_TRACK_NEG = 4, //!< mode for update negative samples - MODE_DETECT = 5 //!< mode for detect samples - }; - - struct CV_EXPORTS Params - { - Params(); - float initInRad; //!< radius for gathering positive instances during init - float trackInPosRad; //!< radius for gathering positive instances during tracking - float searchWinSize; //!< size of search window - int initMaxNegNum; //!< # negative samples to use during init - int trackMaxPosNum; //!< # positive samples to use during training - int trackMaxNegNum; //!< # negative samples to use during training - }; - - /** @brief Constructor - @param parameters TrackerSamplerCSC parameters TrackerSamplerCSC::Params - */ - TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meters = TrackerSamplerCSC::Params() ); - - /** @brief Set the sampling mode of TrackerSamplerCSC - @param samplingMode The sampling mode - - The modes are: - - - "MODE_INIT_POS = 1" -- for the positive sampling in initialization step - - "MODE_INIT_NEG = 2" -- for the negative sampling in initialization step - - "MODE_TRACK_POS = 3" -- for the positive sampling in update step - - "MODE_TRACK_NEG = 4" -- for the negative sampling in update step - - "MODE_DETECT = 5" -- for the sampling in detection step - */ - void setMode( int samplingMode ); - - ~TrackerSamplerCSC(); - - protected: - - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; - - private: - - Params params; - int mode; - RNG rng; - - std::vector sampleImage( const Mat& img, int x, int y, int w, int h, float inrad, float outrad = 0, int maxnum = 1000000 ); -}; - -/** @brief TrackerSampler based on CS (current state), used by algorithm TrackerBoosting - */ -class CV_EXPORTS TrackerSamplerCS : public TrackerSamplerAlgorithm -{ - public: - enum - { - MODE_POSITIVE = 1, //!< mode for positive samples - MODE_NEGATIVE = 2, //!< mode for negative samples - MODE_CLASSIFY = 3 //!< mode for classify samples - }; - - struct CV_EXPORTS Params - { - Params(); - float overlap; //!& sample ) CV_OVERRIDE; - Rect getROI() const; - private: - Rect getTrackingROI( float searchFactor ); - Rect RectMultiply( const Rect & rect, float f ); - std::vector patchesRegularScan( const Mat& image, Rect trackingROI, Size patchSize ); - void setCheckedROI( Rect imageROI ); - - Params params; - int mode; - Rect trackedPatch; - Rect validROI; - Rect ROI; - -}; - -/** @brief This sampler is based on particle filtering. - -In principle, it can be thought of as performing some sort of optimization (and indeed, this -tracker uses opencv's optim module), where tracker seeks to find the rectangle in given frame, -which is the most *"similar"* to the initial rectangle (the one, given through the constructor). - -The optimization performed is stochastic and somehow resembles genetic algorithms, where on each new -image received (submitted via TrackerSamplerPF::sampling()) we start with the region bounded by -boundingBox, then generate several "perturbed" boxes, take the ones most similar to the original. -This selection round is repeated several times. At the end, we hope that only the most promising box -remaining, and these are combined to produce the subrectangle of image, which is put as a sole -element in array sample. - -It should be noted, that the definition of "similarity" between two rectangles is based on comparing -their histograms. As experiments show, tracker is *not* very succesfull if target is assumed to -strongly change its dimensions. - */ -class CV_EXPORTS TrackerSamplerPF : public TrackerSamplerAlgorithm -{ -public: - /** @brief This structure contains all the parameters that can be varied during the course of sampling - algorithm. Below is the structure exposed, together with its members briefly explained with - reference to the above discussion on algorithm's working. - */ - struct CV_EXPORTS Params - { - Params(); - int iterationNum; //!< number of selection rounds - int particlesNum; //!< number of "perturbed" boxes on each round - double alpha; //!< with each new round we exponentially decrease the amount of "perturbing" we allow (like in simulated annealing) - //!< and this very alpha controls how fast annealing happens, ie. how fast perturbing decreases - Mat_ std; //!< initial values for perturbing (1-by-4 array, as each rectangle is given by 4 values -- coordinates of opposite vertices, - //!< hence we have 4 values to perturb) - }; - /** @brief Constructor - @param chosenRect Initial rectangle, that is supposed to contain target we'd like to track. - @param parameters - */ - TrackerSamplerPF(const Mat& chosenRect,const TrackerSamplerPF::Params ¶meters = TrackerSamplerPF::Params()); -protected: - bool samplingImpl( const Mat& image, Rect boundingBox, std::vector& sample ) CV_OVERRIDE; -private: - Params params; - Ptr _solver; - Ptr _function; -}; - -/************************************ Specific TrackerFeature Classes ************************************/ - -/** - * \brief TrackerFeature based on Feature2D - */ -class CV_EXPORTS TrackerFeatureFeature2d : public TrackerFeature -{ - public: - - /** - * \brief Constructor - * \param detectorType string of FeatureDetector - * \param descriptorType string of DescriptorExtractor - */ - TrackerFeatureFeature2d( String detectorType, String descriptorType ); - - ~TrackerFeatureFeature2d() CV_OVERRIDE; - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - - private: - - std::vector keypoints; -}; - -/** - * \brief TrackerFeature based on HOG - */ -class CV_EXPORTS TrackerFeatureHOG : public TrackerFeature -{ - public: - - TrackerFeatureHOG(); - - ~TrackerFeatureHOG() CV_OVERRIDE; - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - -}; - -/** @brief TrackerFeature based on HAAR features, used by TrackerMIL and many others algorithms -@note HAAR features implementation is copied from apps/traincascade and modified according to MIL - */ -class CV_EXPORTS TrackerFeatureHAAR : public TrackerFeature -{ - public: - struct CV_EXPORTS Params - { - Params(); - int numFeatures; //!< # of rects - Size rectSize; //!< rect size - bool isIntegral; //!< true if input images are integral, false otherwise - }; - - /** @brief Constructor - @param parameters TrackerFeatureHAAR parameters TrackerFeatureHAAR::Params - */ - TrackerFeatureHAAR( const TrackerFeatureHAAR::Params ¶meters = TrackerFeatureHAAR::Params() ); - - ~TrackerFeatureHAAR() CV_OVERRIDE; - - /** @brief Compute the features only for the selected indices in the images collection - @param selFeatures indices of selected features - @param images The images - @param response Collection of response for the specific TrackerFeature - */ - bool extractSelected( const std::vector selFeatures, const std::vector& images, Mat& response ); - - /** @brief Identify most effective features - @param response Collection of response for the specific TrackerFeature - @param npoints Max number of features - - @note This method modifies the response parameter - */ - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - /** @brief Swap the feature in position source with the feature in position target - @param source The source position - @param target The target position - */ - bool swapFeature( int source, int target ); - - /** @brief Swap the feature in position id with the feature input - @param id The position - @param feature The feature - */ - bool swapFeature( int id, CvHaarEvaluator::FeatureHaar& feature ); - - /** @brief Get the feature in position id - @param id The position - */ - CvHaarEvaluator::FeatureHaar& getFeatureAt( int id ); - - protected: - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - - private: - - Params params; - Ptr featureEvaluator; -}; - -/** - * \brief TrackerFeature based on LBP - */ -class CV_EXPORTS TrackerFeatureLBP : public TrackerFeature -{ - public: - - TrackerFeatureLBP(); - - ~TrackerFeatureLBP(); - - void selection( Mat& response, int npoints ) CV_OVERRIDE; - - protected: - - bool computeImpl( const std::vector& images, Mat& response ) CV_OVERRIDE; - -}; - /************************************ Specific Tracker Classes ************************************/ /** @brief The MIL algorithm trains a classifier in an online manner to separate the object from the @@ -1062,21 +120,11 @@ based on @cite MIL . Original code can be found here */ -class CV_EXPORTS_W TrackerMIL : public Tracker +class CV_EXPORTS_W TrackerMIL : public cv::legacy::Tracker { public: - struct CV_EXPORTS Params + struct CV_EXPORTS Params : cv::tracking::TrackerMIL::Params { - Params(); - //parameters for sampler - float samplerInitInRadius; //!< radius for gathering positive instances during init - int samplerInitMaxNegNum; //!< # negative samples to use during init - float samplerSearchWinSize; //!< size of search window - float samplerTrackInRadius; //!< radius for gathering positive instances during tracking - int samplerTrackMaxPosNum; //!< # positive samples to use during tracking - int samplerTrackMaxNegNum; //!< # negative samples to use during tracking - int featureSetNumFeatures; //!< # features - void read( const FileNode& fn ); void write( FileStorage& fs ) const; }; @@ -1084,9 +132,9 @@ class CV_EXPORTS_W TrackerMIL : public Tracker /** @brief Constructor @param parameters MIL parameters TrackerMIL::Params */ - static Ptr create(const TrackerMIL::Params ¶meters); + static Ptr create(const TrackerMIL::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerMIL() CV_OVERRIDE {} }; @@ -1097,7 +145,7 @@ This is a real-time object tracking based on a novel on-line version of the AdaB The classifier uses the surrounding background as negative examples in update step to avoid the drifting problem. The implementation is based on @cite OLB . */ -class CV_EXPORTS_W TrackerBoosting : public Tracker +class CV_EXPORTS_W TrackerBoosting : public cv::legacy::Tracker { public: struct CV_EXPORTS Params @@ -1122,9 +170,9 @@ class CV_EXPORTS_W TrackerBoosting : public Tracker /** @brief Constructor @param parameters BOOSTING parameters TrackerBoosting::Params */ - static Ptr create(const TrackerBoosting::Params ¶meters); + static Ptr create(const TrackerBoosting::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerBoosting() CV_OVERRIDE {} }; @@ -1139,7 +187,7 @@ by authors to outperform MIL). During the implementation period the code at , the courtesy of the author Arthur Amarra, was used for the reference purpose. */ -class CV_EXPORTS_W TrackerMedianFlow : public Tracker +class CV_EXPORTS_W TrackerMedianFlow : public cv::legacy::Tracker { public: struct CV_EXPORTS Params @@ -1161,9 +209,9 @@ class CV_EXPORTS_W TrackerMedianFlow : public Tracker /** @brief Constructor @param parameters Median Flow parameters TrackerMedianFlow::Params */ - static Ptr create(const TrackerMedianFlow::Params ¶meters); + static Ptr create(const TrackerMedianFlow::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerMedianFlow() CV_OVERRIDE {} }; @@ -1181,7 +229,7 @@ The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking c implementation, following authors. The tracker is supposed to be able to handle rapid motions, partial occlusions, object absence etc. */ -class CV_EXPORTS_W TrackerTLD : public Tracker +class CV_EXPORTS_W TrackerTLD : public cv::legacy::Tracker { public: struct CV_EXPORTS Params @@ -1194,9 +242,9 @@ class CV_EXPORTS_W TrackerTLD : public Tracker /** @brief Constructor @param parameters TLD parameters TrackerTLD::Params */ - static Ptr create(const TrackerTLD::Params ¶meters); + static Ptr create(const TrackerTLD::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerTLD() CV_OVERRIDE {} }; @@ -1209,7 +257,7 @@ class CV_EXPORTS_W TrackerTLD : public Tracker * as well as the matlab implementation. For more information about KCF with color-names features, please refer to * . */ -class CV_EXPORTS_W TrackerKCF : public Tracker +class CV_EXPORTS_W TrackerKCF : public cv::legacy::Tracker { public: /** @@ -1218,43 +266,12 @@ public: - "GRAY" -- Use grayscale values as the feature - "CN" -- Color-names feature */ - enum MODE { - GRAY = (1 << 0), - CN = (1 << 1), - CUSTOM = (1 << 2) - }; + typedef enum cv::tracking::TrackerKCF::MODE MODE; - struct CV_EXPORTS Params + struct CV_EXPORTS Params : cv::tracking::TrackerKCF::Params { - /** - * \brief Constructor - */ - Params(); - - /** - * \brief Read parameters from a file - */ void read(const FileNode& /*fn*/); - - /** - * \brief Write parameters to a file - */ void write(FileStorage& /*fs*/) const; - - float detect_thresh; //!< detection confidence threshold - float sigma; //!< gaussian kernel bandwidth - float lambda; //!< regularization - float interp_factor; //!< linear interpolation factor for adaptation - float output_sigma_factor; //!< spatial bandwidth (proportional to target) - float pca_learning_rate; //!< compression learning rate - bool resize; //!< activate the resize feature to improve the processing speed - bool split_coeff; //!< split the training coefficients into two matrices - bool wrap_kernel; //!< wrap around the kernel values - bool compress_feature; //!< activate the pca method to compress the features - int max_patch_size; //!< threshold for the ROI size - int compressed_size; //!< feature size after compression - int desc_pca; //!< compressed descriptors of TrackerKCF::MODE - int desc_npca; //!< non-compressed descriptors of TrackerKCF::MODE }; virtual void setFeatureExtractor(void(*)(const Mat, const Rect, Mat&), bool pca_func = false) = 0; @@ -1262,13 +279,14 @@ public: /** @brief Constructor @param parameters KCF parameters TrackerKCF::Params */ - static Ptr create(const TrackerKCF::Params ¶meters); + static Ptr create(const TrackerKCF::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerKCF() CV_OVERRIDE {} }; +#if 0 // legacy variant is not available /** @brief the GOTURN (Generic Object Tracking Using Regression Networks) tracker * GOTURN (@cite GOTURN) is kind of trackers based on Convolutional Neural Networks (CNN). While taking all advantages of CNN trackers, @@ -1284,7 +302,7 @@ public: * * GOTURN architecture goturn.prototxt and trained model goturn.caffemodel are accessible on opencv_extra GitHub repository. */ -class CV_EXPORTS_W TrackerGOTURN : public Tracker +class CV_EXPORTS_W TrackerGOTURN : public cv::legacy::Tracker { public: struct CV_EXPORTS Params @@ -1299,12 +317,13 @@ public: /** @brief Constructor @param parameters GOTURN parameters TrackerGOTURN::Params */ - static Ptr create(const TrackerGOTURN::Params ¶meters); + static Ptr create(const TrackerGOTURN::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerGOTURN() CV_OVERRIDE {} }; +#endif /** @brief the MOSSE (Minimum Output Sum of Squared %Error) tracker @@ -1312,12 +331,12 @@ The implementation is based on @cite MOSSE Visual Object Tracking using Adaptive @note this tracker works with grayscale images, if passed bgr ones, they will get converted internally. */ -class CV_EXPORTS_W TrackerMOSSE : public Tracker +class CV_EXPORTS_W TrackerMOSSE : public cv::legacy::Tracker { public: /** @brief Constructor */ - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); virtual ~TrackerMOSSE() CV_OVERRIDE {} }; @@ -1350,7 +369,7 @@ public: * @param image input image * @param boundingBox a rectangle represents ROI of the tracked object */ - CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); + CV_WRAP bool add(Ptr newTracker, InputArray image, const Rect2d& boundingBox); /** * \brief Add a set of objects to be tracked. @@ -1358,7 +377,7 @@ public: * @param image input image * @param boundingBox list of the tracked objects */ - bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); + bool add(std::vector > newTrackers, InputArray image, std::vector boundingBox); /** * \brief Update the current tracking status. @@ -1415,7 +434,7 @@ public: @return True if new target initialization went succesfully, false otherwise */ - bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); + bool addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm); /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets @param image The current frame @@ -1478,16 +497,11 @@ public: The implementation is based on @cite Lukezic_IJCV2018 Discriminative Correlation Filter with Channel and Spatial Reliability */ -class CV_EXPORTS_W TrackerCSRT : public Tracker +class CV_EXPORTS_W TrackerCSRT : public cv::legacy::Tracker { public: - struct CV_EXPORTS Params + struct CV_EXPORTS Params : cv::tracking::TrackerCSRT::Params { - /** - * \brief Constructor - */ - Params(); - /** * \brief Read parameters from a file */ @@ -1497,52 +511,28 @@ public: * \brief Write parameters to a file */ void write(cv::FileStorage& fs) const; - - bool use_hog; - bool use_color_names; - bool use_gray; - bool use_rgb; - bool use_channel_weights; - bool use_segmentation; - - std::string window_function; //!< Window function: "hann", "cheb", "kaiser" - float kaiser_alpha; - float cheb_attenuation; - - float template_size; - float gsl_sigma; - float hog_orientations; - float hog_clip; - float padding; - float filter_lr; - float weights_lr; - int num_hog_channels_used; - int admm_iterations; - int histogram_bins; - float histogram_lr; - int background_ratio; - int number_of_scales; - float scale_sigma_factor; - float scale_model_max_area; - float scale_lr; - float scale_step; - - float psr_threshold; //!< we lost the target, if the psr is lower than this. }; /** @brief Constructor @param parameters CSRT parameters TrackerCSRT::Params */ - static Ptr create(const TrackerCSRT::Params ¶meters); + static Ptr create(const TrackerCSRT::Params ¶meters); - CV_WRAP static Ptr create(); + CV_WRAP static Ptr create(); CV_WRAP virtual void setInitialMask(InputArray mask) = 0; virtual ~TrackerCSRT() CV_OVERRIDE {} }; + +CV_EXPORTS_W Ptr upgradeTrackingAPI(const Ptr& legacy_tracker); + //! @} -} /* namespace cv */ +#ifndef CV_DOXYGEN +} // namespace #endif +}} // namespace + +#endif // OPENCV_TRACKING_LEGACY_HPP diff --git a/modules/tracking/misc/java/gen_dict.json b/modules/tracking/misc/java/gen_dict.json new file mode 100644 index 000000000..a2c3bf0da --- /dev/null +++ b/modules/tracking/misc/java/gen_dict.json @@ -0,0 +1,5 @@ +{ + "namespaces_dict": { + "cv.legacy": "legacy" + } +} diff --git a/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java b/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java new file mode 100644 index 000000000..3c8bfa999 --- /dev/null +++ b/modules/tracking/misc/java/test/TrackerCreateLegacyTest.java @@ -0,0 +1,23 @@ +package org.opencv.test.tracking; + +import org.opencv.core.Core; +import org.opencv.core.CvException; +import org.opencv.test.OpenCVTestCase; + +import org.opencv.tracking.Tracking; +import org.opencv.tracking.legacy_Tracker; +import org.opencv.tracking.legacy_TrackerTLD; + +public class TrackerCreateLegacyTest extends OpenCVTestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + + public void testCreateLegacyTrackerTLD() { + legacy_Tracker tracker = legacy_TrackerTLD.create(); + } + +} diff --git a/modules/tracking/misc/java/test/TrackerCreateTest.java b/modules/tracking/misc/java/test/TrackerCreateTest.java new file mode 100644 index 000000000..189ccfcd5 --- /dev/null +++ b/modules/tracking/misc/java/test/TrackerCreateTest.java @@ -0,0 +1,38 @@ +package org.opencv.test.tracking; + +import org.opencv.core.Core; +import org.opencv.core.CvException; +import org.opencv.test.OpenCVTestCase; + +import org.opencv.tracking.Tracking; +import org.opencv.tracking.Tracker; +import org.opencv.tracking.TrackerGOTURN; +import org.opencv.tracking.TrackerKCF; +import org.opencv.tracking.TrackerMIL; + +public class TrackerCreateTest extends OpenCVTestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + + public void testCreateTrackerGOTURN() { + try { + Tracker tracker = TrackerGOTURN.create(); + assert(tracker != null); + } catch (CvException e) { + // expected, model files may be missing + } + } + + public void testCreateTrackerKCF() { + Tracker tracker = TrackerKCF.create(); + } + + public void testCreateTrackerMIL() { + Tracker tracker = TrackerMIL.create(); + } + +} diff --git a/modules/tracking/misc/objc/gen_dict.json b/modules/tracking/misc/objc/gen_dict.json index 6cefc5f55..7172e2df4 100644 --- a/modules/tracking/misc/objc/gen_dict.json +++ b/modules/tracking/misc/objc/gen_dict.json @@ -4,5 +4,8 @@ }, "AdditionalImports" : { "*" : [ "\"tracking.hpp\"" ] - } + }, + "namespace_ignore_list" : [ + "cv.legacy" + ] } diff --git a/modules/tracking/misc/python/pyopencv_tracking.hpp b/modules/tracking/misc/python/pyopencv_tracking.hpp new file mode 100644 index 000000000..dd4d52696 --- /dev/null +++ b/modules/tracking/misc/python/pyopencv_tracking.hpp @@ -0,0 +1,6 @@ +#ifdef HAVE_OPENCV_TRACKING +typedef TrackerCSRT::Params TrackerCSRT_Params; +typedef TrackerKCF::Params TrackerKCF_Params; +typedef TrackerMIL::Params TrackerMIL_Params; +typedef TrackerGOTURN::Params TrackerGOTURN_Params; +#endif diff --git a/modules/tracking/misc/python/test/test_tracking_contrib.py b/modules/tracking/misc/python/test/test_tracking_contrib.py new file mode 100644 index 000000000..7eeb91e1e --- /dev/null +++ b/modules/tracking/misc/python/test/test_tracking_contrib.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +import os +import numpy as np +import cv2 as cv + +from tests_common import NewOpenCVTests, unittest + +class tracking_contrib_test(NewOpenCVTests): + + def test_createTracker(self): + + t = cv.TrackerMIL_create() + t = cv.TrackerKCF_create() + try: + t = cv.TrackerGOTURN_create() + except cv.error as e: + pass # may fail due to missing DL model files + + def test_createLegacyTracker(self): + + t = cv.legacy.TrackerBoosting_create() + t = cv.legacy.TrackerMIL_create() + t = cv.legacy.TrackerKCF_create() + t = cv.legacy.TrackerMedianFlow_create() + #t = cv.legacy.TrackerGOTURN_create() + t = cv.legacy.TrackerMOSSE_create() + t = cv.legacy.TrackerCSRT_create() + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/tracking/perf/perf_Tracker.cpp b/modules/tracking/perf/perf_Tracker.cpp deleted file mode 100644 index 4667c713b..000000000 --- a/modules/tracking/perf/perf_Tracker.cpp +++ /dev/null @@ -1,412 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "perf_precomp.hpp" - -namespace opencv_test { namespace { - -//write sanity: ./bin/opencv_perf_tracking --perf_write_sanity=true --perf_min_samples=1 -//verify sanity: ./bin/opencv_perf_tracking --perf_min_samples=1 - -#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") -//#define TESTSET_NAMES testing::internal::ValueArray1("david") -#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - -const string TRACKING_DIR = "cv/tracking"; -const string FOLDER_IMG = "data"; - -typedef perf::TestBaseWithParam > tracking; - -std::vector splitString( std::string s, std::string delimiter ) -{ - std::vector token; - size_t pos = 0; - while ( ( pos = s.find( delimiter ) ) != std::string::npos ) - { - token.push_back( s.substr( 0, pos ) ); - s.erase( 0, pos + delimiter.length() ); - } - token.push_back( s ); - return token; -} - -void checkData( const string& datasetMeta, int& startFrame, string& prefix, string& suffix ) -{ - //get informations on the current test data - FileStorage fs; - fs.open( datasetMeta, FileStorage::READ ); - fs["start"] >> startFrame; - fs["prefix"] >> prefix; - fs["suffix"] >> suffix; - fs.release(); -} - -bool getGroundTruth( const string& gtFile, vector& gtBBs ) -{ - std::ifstream gt; - //open the ground truth - gt.open( gtFile.c_str() ); - if( !gt.is_open() ) - { - return false; - } - string line; - Rect currentBB; - while ( getline( gt, line ) ) - { - vector tokens = splitString( line, "," ); - - if( tokens.size() != 4 ) - { - return false; - } - - gtBBs.push_back( - Rect( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ) ); - } - return true; -} - -void getSegment( int segmentId, int numSegments, int bbCounter, int& startFrame, int& endFrame ) -{ - //compute the start and the and for each segment - int gtStartFrame = startFrame; - int numFrame = bbCounter / numSegments; - startFrame += ( segmentId - 1 ) * numFrame; - endFrame = startFrame + numFrame; - - if( ( segmentId ) == numSegments ) - endFrame = bbCounter + gtStartFrame - 1; -} - -void getMatOfRects( const vector& bbs, Mat& bbs_mat ) -{ - for ( int b = 0, size = (int)bbs.size(); b < size; b++ ) - { - bbs_mat.at( b, 0 ) = (float)bbs[b].x; - bbs_mat.at( b, 1 ) = (float)bbs[b].y; - bbs_mat.at( b, 2 ) = (float)bbs[b].width; - bbs_mat.at( b, 3 ) = (float)bbs[b].height; - } -} - -PERF_TEST_P(tracking, mil, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerMIL::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, boosting, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerBoosting::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, tld, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>( GetParam() ); - int segmentId = get<1>( GetParam() ); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); - checkData( datasetMeta, startFrame, prefix, suffix ); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); - if( !getGroundTruth( gtFile, gtBBs ) ) - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerTLD::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); - int endFrame = 0; - getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); - for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) - { - c >> frame; - - if( frame.empty() ) - { - break; - } - - if( !initialized ) - { - if( !tracker->init( frame, currentBB ) ) - { - FAIL()<< "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if( initialized ) - { - tracker->update( frame, currentBB ); - } - bbs.push_back( currentBB ); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat( (int)bbs.size(), 4, CV_32F ); - getMatOfRects( bbs, bbs_mat ); - - SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); - -} - -PERF_TEST_P(tracking, GOTURN, testing::Combine(TESTSET_NAMES, SEGMENTS)) -{ - string video = get<0>(GetParam()); - int segmentId = get<1>(GetParam()); - - int startFrame; - string prefix; - string suffix; - string datasetMeta = getDataPath(TRACKING_DIR + "/" + video + "/" + video + ".yml"); - checkData(datasetMeta, startFrame, prefix, suffix); - int gtStartFrame = startFrame; - - vector gtBBs; - string gtFile = getDataPath(TRACKING_DIR + "/" + video + "/gt.txt"); - if (!getGroundTruth(gtFile, gtBBs)) - FAIL() << "Ground truth file " << gtFile << " can not be read" << endl; - int bbCounter = (int)gtBBs.size(); - - Mat frame; - bool initialized = false; - vector bbs; - - Ptr tracker = TrackerGOTURN::create(); - string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; - int numSegments = (sizeof(SEGMENTS) / sizeof(int)); - int endFrame = 0; - getSegment(segmentId, numSegments, bbCounter, startFrame, endFrame); - - Rect currentBBi = gtBBs[startFrame - gtStartFrame]; - Rect2d currentBB(currentBBi); - - TEST_CYCLE_N(1) - { - VideoCapture c; - c.open(getDataPath(TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm")); - c.set(CAP_PROP_POS_FRAMES, startFrame); - for (int frameCounter = startFrame; frameCounter < endFrame; frameCounter++) - { - c >> frame; - - if (frame.empty()) - { - break; - } - - if (!initialized) - { - if (!tracker->init(frame, currentBB)) - { - FAIL() << "Could not initialize tracker" << endl; - return; - } - initialized = true; - } - else if (initialized) - { - tracker->update(frame, currentBB); - } - bbs.push_back(currentBB); - - } - } - //save the bounding boxes in a Mat - Mat bbs_mat((int)bbs.size(), 4, CV_32F); - getMatOfRects(bbs, bbs_mat); - - SANITY_CHECK(bbs_mat, 15, ERROR_RELATIVE); - -} - -}} // namespace diff --git a/modules/tracking/perf/perf_main.cpp b/modules/tracking/perf/perf_main.cpp index cfd91a4dc..ccca051e6 100644 --- a/modules/tracking/perf/perf_main.cpp +++ b/modules/tracking/perf/perf_main.cpp @@ -3,4 +3,19 @@ // of this distribution and at http://opencv.org/license.html. #include "perf_precomp.hpp" -CV_PERF_TEST_MAIN(tracking) +static +void initTrackingTests() +{ + const char* extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("OPENCV_DNN_TEST_DATA_PATH"); +#endif + if (extraTestDataPath) + cvtest::addDataSearchPath(extraTestDataPath); + + cvtest::addDataSearchSubDirectory(""); // override "cv" prefix below to access without "../dnn" hacks +} + +CV_PERF_TEST_MAIN(tracking, initTrackingTests()) diff --git a/modules/tracking/perf/perf_trackers.cpp b/modules/tracking/perf/perf_trackers.cpp new file mode 100644 index 000000000..e76b15b2c --- /dev/null +++ b/modules/tracking/perf/perf_trackers.cpp @@ -0,0 +1,119 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" + +#include + +namespace opencv_test { namespace { +using namespace perf; + +//using namespace cv::tracking; + +typedef tuple TrackingParams_t; + +std::vector getTrackingParams() +{ + std::vector params { + TrackingParams_t("david/data/david.webm", 300, Rect(163,62,47,56)), + TrackingParams_t("dudek/data/dudek.webm", 1, Rect(123,87,132,176)), + TrackingParams_t("faceocc2/data/faceocc2.webm", 1, Rect(118,57,82,98)) + }; + return params; +} + +class Tracking : public perf::TestBaseWithParam +{ +public: + template + void runTrackingTest(const Ptr& tracker, const TrackingParams_t& params); +}; + +template +void Tracking::runTrackingTest(const Ptr& tracker, const TrackingParams_t& params) +{ + const int N = 10; + string video = get<0>(params); + int startFrame = get<1>(params); + //int endFrame = startFrame + N; + Rect boundingBox = get<2>(params); + + string videoPath = findDataFile(std::string("cv/tracking/") + video); + + VideoCapture c; + c.open(videoPath); + ASSERT_TRUE(c.isOpened()) << videoPath; +#if 0 + // c.set(CAP_PROP_POS_FRAMES, startFrame); +#else + if (startFrame) + std::cout << "startFrame = " << startFrame << std::endl; + for (int i = 0; i < startFrame; i++) + { + Mat dummy_frame; + c >> dummy_frame; + ASSERT_FALSE(dummy_frame.empty()) << i << ": " << videoPath; + } +#endif + + // decode frames into memory (don't measure decoding performance) + std::vector frames; + for (int i = 0; i < N; ++i) + { + Mat frame; + c >> frame; + ASSERT_FALSE(frame.empty()) << "i=" << i; + frames.push_back(frame); + } + + std::cout << "frame size = " << frames[0].size() << std::endl; + + PERF_SAMPLE_BEGIN(); + { + tracker->init(frames[0], (ROI_t)boundingBox); + for (int i = 1; i < N; ++i) + { + ROI_t rc; + tracker->update(frames[i], rc); + ASSERT_FALSE(rc.empty()); + } + } + PERF_SAMPLE_END(); + + SANITY_CHECK_NOTHING(); +} + + +//================================================================================================== + +PERF_TEST_P(Tracking, MIL, testing::ValuesIn(getTrackingParams())) +{ + auto tracker = TrackerMIL::create(); + runTrackingTest(tracker, GetParam()); +} + +PERF_TEST_P(Tracking, Boosting, testing::ValuesIn(getTrackingParams())) +{ + auto tracker = legacy::TrackerBoosting::create(); + runTrackingTest(tracker, GetParam()); +} + +PERF_TEST_P(Tracking, TLD, testing::ValuesIn(getTrackingParams())) +{ + auto tracker = legacy::TrackerTLD::create(); + runTrackingTest(tracker, GetParam()); +} + +PERF_TEST_P(Tracking, GOTURN, testing::ValuesIn(getTrackingParams())) +{ + std::string model = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.prototxt"); + std::string weights = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.caffemodel", false); + TrackerGOTURN::Params params; + params.modelTxt = model; + params.modelBin = weights; + auto tracker = TrackerGOTURN::create(params); + runTrackingTest(tracker, GetParam()); +} + +}} // namespace diff --git a/modules/tracking/samples/benchmark.cpp b/modules/tracking/samples/benchmark.cpp index 7becc6a03..935244f92 100644 --- a/modules/tracking/samples/benchmark.cpp +++ b/modules/tracking/samples/benchmark.cpp @@ -93,7 +93,7 @@ struct AlgoWrap Ptr tracker; bool lastRes; - Rect2d lastBox; + Rect lastBox; State lastState; // visual @@ -112,14 +112,14 @@ struct AlgoWrap void eval(const Mat &frame, const Rect2d >Box, bool isVerbose) { // RUN - lastBox = Rect2d(); + lastBox = Rect(); int64 frameTime = getTickCount(); lastRes = tracker->update(frame, lastBox); frameTime = getTickCount() - frameTime; // RESULTS - double intersectArea = (gtBox & lastBox).area(); - double unionArea = (gtBox | lastBox).area(); + double intersectArea = (gtBox & (Rect2d)lastBox).area(); + double unionArea = (gtBox | (Rect2d)lastBox).area(); numTotal++; numResponse += (lastRes && isGoodBox(lastBox)) ? 1 : 0; numPresent += isGoodBox(gtBox) ? 1 : 0; diff --git a/modules/tracking/samples/csrt.cpp b/modules/tracking/samples/csrt.cpp index 1f77e89a8..b52c8d37a 100644 --- a/modules/tracking/samples/csrt.cpp +++ b/modules/tracking/samples/csrt.cpp @@ -41,7 +41,7 @@ int main(int argc, char** argv) cap >> frame; // target bounding box - Rect2d roi; + Rect roi; if (argc > 2) { // read first line of ground-truth file std::string groundtruthPath = argv[2]; diff --git a/modules/tracking/samples/goturnTracker.cpp b/modules/tracking/samples/goturnTracker.cpp index a1e721718..9bdaa97a1 100644 --- a/modules/tracking/samples/goturnTracker.cpp +++ b/modules/tracking/samples/goturnTracker.cpp @@ -50,6 +50,7 @@ #include "opencv2/datasets/track_alov.hpp" #include +#include #include #include #include @@ -65,7 +66,7 @@ static Mat image; static bool paused; static bool selectObjects = false; static bool startSelection = false; -Rect2d boundingBox; +static Rect boundingBox; static const char* keys = { "{@dataset_path || Dataset path }" @@ -140,7 +141,7 @@ int main(int argc, char *argv[]) setMouseCallback("GOTURN Tracking", onMouse, 0); //Create GOTURN tracker - Ptr tracker = TrackerGOTURN::create(); + auto tracker = TrackerGOTURN::create(); //Load and init full ALOV300++ dataset with a given datasetID, as alternative you can use loadAnnotatedOnly(..) //to load only frames with labelled ground truth ~ every 5-th frame @@ -181,11 +182,7 @@ int main(int argc, char *argv[]) if (!initialized && selectObjects) { //Initialize the tracker and add targets - if (!tracker->init(frame, boundingBox)) - { - cout << "Tracker Init Error!!!"; - return 0; - } + tracker->init(frame, boundingBox); rectangle(frame, boundingBox, Scalar(0, 0, 255), 2, 1); initialized = true; } diff --git a/modules/tracking/samples/kcf.cpp b/modules/tracking/samples/kcf.cpp index f40a3c5f2..e1d7ab207 100644 --- a/modules/tracking/samples/kcf.cpp +++ b/modules/tracking/samples/kcf.cpp @@ -41,7 +41,7 @@ int main( int argc, char** argv ){ // get bounding box cap >> frame; - Rect2d roi= selectROI("tracker", frame, true, false); + Rect roi = selectROI("tracker", frame, true, false); //quit if ROI was not selected if(roi.width==0 || roi.height==0) diff --git a/modules/tracking/samples/multiTracker_dataset.cpp b/modules/tracking/samples/multiTracker_dataset.cpp index 622a69ac7..efaed5633 100644 --- a/modules/tracking/samples/multiTracker_dataset.cpp +++ b/modules/tracking/samples/multiTracker_dataset.cpp @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) namedWindow("Tracking API", 0); setMouseCallback("Tracking API", onMouse, 0); - MultiTrackerTLD mt; + legacy::MultiTrackerTLD mt; //Init Dataset Ptr dataset = TRACK_vot::create(); dataset->load(datasetRootPath); @@ -185,7 +185,7 @@ int main(int argc, char *argv[]) //Initialize the tracker and add targets for (int i = 0; i < (int)boundingBoxes.size(); i++) { - if (!mt.addTarget(frame, boundingBoxes[i], createTrackerByName(tracker_algorithm))) + if (!mt.addTarget(frame, boundingBoxes[i], createTrackerByName_legacy(tracker_algorithm))) { cout << "Trackers Init Error!!!"; return 0; diff --git a/modules/tracking/samples/multitracker.cpp b/modules/tracking/samples/multitracker.cpp index 7b21334d5..62df01991 100644 --- a/modules/tracking/samples/multitracker.cpp +++ b/modules/tracking/samples/multitracker.cpp @@ -73,7 +73,7 @@ int main( int argc, char** argv ){ trackingAlg = argv[2]; // create the tracker - MultiTracker trackers; + legacy::MultiTracker trackers; // container of the tracked objects vector ROIs; @@ -93,10 +93,10 @@ int main( int argc, char** argv ){ if(ROIs.size()<1) return 0; - std::vector > algorithms; + std::vector > algorithms; for (size_t i = 0; i < ROIs.size(); i++) { - algorithms.push_back(createTrackerByName(trackingAlg)); + algorithms.push_back(createTrackerByName_legacy(trackingAlg)); objects.push_back(ROIs[i]); } diff --git a/modules/tracking/samples/samples_utility.hpp b/modules/tracking/samples/samples_utility.hpp index de3525d50..2fe876919 100644 --- a/modules/tracking/samples/samples_utility.hpp +++ b/modules/tracking/samples/samples_utility.hpp @@ -2,25 +2,28 @@ #define _SAMPLES_UTILITY_HPP_ #include +#include -inline cv::Ptr createTrackerByName(cv::String name) +inline cv::Ptr createTrackerByName(const std::string& name) { + using namespace cv; + cv::Ptr tracker; if (name == "KCF") tracker = cv::TrackerKCF::create(); else if (name == "TLD") - tracker = cv::TrackerTLD::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerTLD::create()); else if (name == "BOOSTING") - tracker = cv::TrackerBoosting::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerBoosting::create()); else if (name == "MEDIAN_FLOW") - tracker = cv::TrackerMedianFlow::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerMedianFlow::create()); else if (name == "MIL") tracker = cv::TrackerMIL::create(); else if (name == "GOTURN") tracker = cv::TrackerGOTURN::create(); else if (name == "MOSSE") - tracker = cv::TrackerMOSSE::create(); + tracker = legacy::upgradeTrackingAPI(legacy::TrackerMOSSE::create()); else if (name == "CSRT") tracker = cv::TrackerCSRT::create(); else @@ -29,4 +32,32 @@ inline cv::Ptr createTrackerByName(cv::String name) return tracker; } +inline cv::Ptr createTrackerByName_legacy(const std::string& name) +{ + using namespace cv; + + cv::Ptr tracker; + + if (name == "KCF") + tracker = legacy::TrackerKCF::create(); + else if (name == "TLD") + tracker = legacy::TrackerTLD::create(); + else if (name == "BOOSTING") + tracker = legacy::TrackerBoosting::create(); + else if (name == "MEDIAN_FLOW") + tracker = legacy::TrackerMedianFlow::create(); + else if (name == "MIL") + tracker = legacy::TrackerMIL::create(); + else if (name == "GOTURN") + CV_Error(cv::Error::StsNotImplemented, "FIXIT: migration on new API is required"); + else if (name == "MOSSE") + tracker = legacy::TrackerMOSSE::create(); + else if (name == "CSRT") + tracker = legacy::TrackerCSRT::create(); + else + CV_Error(cv::Error::StsBadArg, "Invalid tracking algorithm name\n"); + + return tracker; +} + #endif diff --git a/modules/tracking/samples/tracker.cpp b/modules/tracking/samples/tracker.cpp index add014c8a..5de5e60f6 100644 --- a/modules/tracking/samples/tracker.cpp +++ b/modules/tracking/samples/tracker.cpp @@ -88,7 +88,7 @@ int main( int argc, char** argv ){ namedWindow( "Tracking API", 1 ); Mat image; - Rect2d boundingBox; + Rect boundingBox; bool paused = false; //instantiates the specific Tracker @@ -134,11 +134,7 @@ int main( int argc, char** argv ){ if( !initialized ) { //initializes the tracker - if( !tracker->init( frame, boundingBox ) ) - { - cout << "***Could not initialize tracker...***\n"; - return -1; - } + tracker->init(frame, boundingBox); initialized = true; } else if( initialized ) diff --git a/modules/tracking/samples/tracker_dataset.cpp b/modules/tracking/samples/tracker_dataset.cpp index 1dbeb4c84..479546bb7 100644 --- a/modules/tracking/samples/tracker_dataset.cpp +++ b/modules/tracking/samples/tracker_dataset.cpp @@ -66,7 +66,7 @@ using namespace cv::datasets; //#define RECORD_VIDEO_FLG static Mat image; -static Rect2d boundingBox; +static Rect boundingBox; static bool paused; static bool selectObject = false; static bool startSelection = false; @@ -186,11 +186,7 @@ int main(int argc, char *argv[]) if (!initialized && selectObject) { //initializes the tracker - if (!tracker->init(frame, boundingBox)) - { - cout << "***Could not initialize tracker...***\n"; - return -1; - } + tracker->init(frame, boundingBox); initialized = true; } else if (initialized) diff --git a/modules/tracking/samples/tracking_by_matching.cpp b/modules/tracking/samples/tracking_by_matching.cpp index 1e53845a1..d2e10afd6 100644 --- a/modules/tracking/samples/tracking_by_matching.cpp +++ b/modules/tracking/samples/tracking_by_matching.cpp @@ -8,7 +8,8 @@ using namespace std; using namespace cv; -using namespace cv::tbm; +using namespace cv::detail::tracking; +using namespace cv::detail::tracking::tbm; static const char* keys = { "{video_name | | video name }" @@ -123,7 +124,7 @@ private: cv::Ptr createTrackerByMatchingWithFastDescriptor() { - cv::tbm::TrackerParams params; + tbm::TrackerParams params; cv::Ptr tracker = createTrackerByMatching(params); diff --git a/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp b/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp index cd4418b2d..58e2beb03 100644 --- a/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp +++ b/modules/tracking/samples/tutorial_customizing_cn_tracker.cpp @@ -25,7 +25,7 @@ int main( int argc, char** argv ){ } // declares all required variables - Rect2d roi; + Rect roi; Mat frame; //! [param] diff --git a/modules/tracking/samples/tutorial_introduction_to_tracker.cpp b/modules/tracking/samples/tutorial_introduction_to_tracker.cpp index e092c7d99..4083c32d3 100644 --- a/modules/tracking/samples/tutorial_introduction_to_tracker.cpp +++ b/modules/tracking/samples/tutorial_introduction_to_tracker.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -24,7 +25,7 @@ int main( int argc, char** argv ){ // declares all required variables //! [vars] - Rect2d roi; + Rect roi; Mat frame; //! [vars] diff --git a/modules/tracking/samples/tutorial_multitracker.cpp b/modules/tracking/samples/tutorial_multitracker.cpp index 0c6d21800..efb9af881 100644 --- a/modules/tracking/samples/tutorial_multitracker.cpp +++ b/modules/tracking/samples/tutorial_multitracker.cpp @@ -40,7 +40,7 @@ int main( int argc, char** argv ){ // create the tracker //! [create] - MultiTracker trackers; + legacy::MultiTracker trackers; //! [create] // container of the tracked objects @@ -67,10 +67,10 @@ int main( int argc, char** argv ){ // initialize the tracker //! [init] - std::vector > algorithms; + std::vector > algorithms; for (size_t i = 0; i < ROIs.size(); i++) { - algorithms.push_back(createTrackerByName(trackingAlg)); + algorithms.push_back(createTrackerByName_legacy(trackingAlg)); objects.push_back(ROIs[i]); } diff --git a/modules/tracking/src/PFSolver.hpp b/modules/tracking/src/PFSolver.hpp index 520fa5f7e..55c5b8193 100644 --- a/modules/tracking/src/PFSolver.hpp +++ b/modules/tracking/src/PFSolver.hpp @@ -1,5 +1,3 @@ -#include "opencv2/core.hpp" -#include "opencv2/core/core_c.h" #include #include #include diff --git a/modules/tracking/src/augmented_unscented_kalman.cpp b/modules/tracking/src/augmented_unscented_kalman.cpp index 85fdfabc4..efc060b02 100644 --- a/modules/tracking/src/augmented_unscented_kalman.cpp +++ b/modules/tracking/src/augmented_unscented_kalman.cpp @@ -42,10 +42,10 @@ #include "precomp.hpp" #include "opencv2/tracking/kalman_filters.hpp" -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { +inline namespace kalman_filters { void AugmentedUnscentedKalmanFilterParams:: init( int dp, int mp, int cp, double processNoiseCovDiag, double measurementNoiseCovDiag, @@ -394,5 +394,4 @@ Ptr createAugmentedUnscentedKalmanFilter(const AugmentedU return kfu; } -} // tracking -} // cv +}}}} // namespace diff --git a/modules/tracking/src/feature.cpp b/modules/tracking/src/feature.cpp index 7b8bfcff7..ecbceb572 100644 --- a/modules/tracking/src/feature.cpp +++ b/modules/tracking/src/feature.cpp @@ -42,8 +42,9 @@ #include "precomp.hpp" #include "opencv2/tracking/feature.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TODO This implementation is based on apps/traincascade/ @@ -309,14 +310,14 @@ void CvHaarEvaluator::FeatureHaar::generateRandomFeature( Size patchSize ) position.y = rand() % ( patchSize.height ); position.x = rand() % ( patchSize.width ); - baseDim.width = (int) ( ( 1 - sqrt( 1 - (float) rand() / RAND_MAX ) ) * patchSize.width ); - baseDim.height = (int) ( ( 1 - sqrt( 1 - (float) rand() / RAND_MAX ) ) * patchSize.height ); + baseDim.width = (int) ( ( 1 - sqrt( 1 - (float) rand() * (float)(1.0 / RAND_MAX) ) ) * patchSize.width ); + baseDim.height = (int) ( ( 1 - sqrt( 1 - (float) rand() * (float)(1.0 / RAND_MAX) ) ) * patchSize.height ); //select types //float probType[11] = {0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0909f, 0.0950f}; float probType[11] = { 0.2f, 0.2f, 0.2f, 0.2f, 0.2f, 0.2f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; - float prob = (float) rand() / RAND_MAX; + float prob = (float) rand() * (float)(1.0 / RAND_MAX); if( prob < probType[0] ) { @@ -686,7 +687,7 @@ void CvHaarEvaluator::FeatureHaar::generateRandomFeature( Size patchSize ) valid = true; } else - CV_Error(CV_StsAssert, ""); + CV_Error(Error::StsAssert, ""); } m_initSize = patchSize; @@ -1069,4 +1070,4 @@ void CvLBPEvaluator::Feature::write( FileStorage &fs ) const fs << CC_RECT << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/featureColorName.cpp b/modules/tracking/src/featureColorName.cpp index cb3aa3cf4..48bc360a5 100644 --- a/modules/tracking/src/featureColorName.cpp +++ b/modules/tracking/src/featureColorName.cpp @@ -42,7 +42,10 @@ #include "precomp.hpp" #include -namespace cv{ +namespace cv { +namespace detail { +inline namespace tracking { + const float ColorNames[][10]={ {0.45975f,0.014802f,0.044289f,-0.028193f,0.001151f,-0.0050145f,0.34522f,0.018362f,0.23994f,0.1689f}, {0.47157f,0.021424f,0.041444f,-0.030215f,0.0019002f,-0.0029264f,0.32875f,0.0082059f,0.2502f,0.17007f}, @@ -32813,4 +32816,5 @@ namespace cv{ {0.0030858f,-0.016151f,0.013017f,0.0072284f,-0.53357f,0.30985f,0.0041336f,-0.012531f,0.00142f,-0.33842f}, {0.0087778f,-0.015645f,0.004769f,0.011785f,-0.54199f,0.31505f,0.00020476f,-0.020282f,0.00021236f,-0.34675f} }; -} + +}}} // namespace diff --git a/modules/tracking/src/gtrTracker.cpp b/modules/tracking/src/gtrTracker.cpp index c56bb5bba..b6dc1e004 100644 --- a/modules/tracking/src/gtrTracker.cpp +++ b/modules/tracking/src/gtrTracker.cpp @@ -38,96 +38,75 @@ // the use of this software, even if advised of the possibility of such damage. // //M*/ -#include "opencv2/opencv_modules.hpp" -#include "gtrTracker.hpp" +#include "precomp.hpp" +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#endif -namespace cv -{ +namespace cv { +inline namespace tracking { -TrackerGOTURN::Params::Params() +TrackerGOTURN::TrackerGOTURN() { - modelTxt = "goturn.prototxt"; - modelBin = "goturn.caffemodel"; + // nothing } -void TrackerGOTURN::Params::read(const cv::FileNode& /*fn*/){} - -void TrackerGOTURN::Params::write(cv::FileStorage& /*fs*/) const {} - - -Ptr TrackerGOTURN::create(const TrackerGOTURN::Params ¶meters) +TrackerGOTURN::~TrackerGOTURN() { -#ifdef HAVE_OPENCV_DNN - return Ptr(new gtr::TrackerGOTURNImpl(parameters)); -#else - (void)(parameters); - CV_Error(cv::Error::StsNotImplemented , "to use GOTURN, the tracking module needs to be built with opencv_dnn !"); -#endif + // nothing } -Ptr TrackerGOTURN::create() + +TrackerGOTURN::Params::Params() { - return TrackerGOTURN::create(TrackerGOTURN::Params()); + modelTxt = "goturn.prototxt"; + modelBin = "goturn.caffemodel"; } - #ifdef HAVE_OPENCV_DNN -namespace gtr -{ -class TrackerGOTURNModel : public TrackerModel{ +class TrackerGOTURNImpl : public TrackerGOTURN +{ public: - TrackerGOTURNModel(TrackerGOTURN::Params){} - Rect2d getBoundingBox(){ return boundingBox_; } - void setBoudingBox(Rect2d boundingBox) { + TrackerGOTURNImpl(const TrackerGOTURN::Params ¶meters) + : params(parameters) + { + // Load GOTURN architecture from *.prototxt and pretrained weights from *.caffemodel + net = dnn::readNetFromCaffe(params.modelTxt, params.modelBin); + CV_Assert(!net.empty()); + } + + void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + + void setBoudingBox(Rect boundingBox) + { if (image_.empty()) CV_Error(Error::StsInternal, "Set image first"); - boundingBox_ = boundingBox & Rect2d(Point(0, 0), image_.size()); + boundingBox_ = boundingBox & Rect(Point(0, 0), image_.size()); } - Mat getImage(){ return image_; } - void setImage(const Mat& image){ image.copyTo(image_); } -protected: - Rect2d boundingBox_; - Mat image_; - void modelEstimationImpl(const std::vector&) CV_OVERRIDE {} - void modelUpdateImpl() CV_OVERRIDE {} -}; - -TrackerGOTURNImpl::TrackerGOTURNImpl(const TrackerGOTURN::Params ¶meters) : - params(parameters){ - isInit = false; -}; -void TrackerGOTURNImpl::read(const cv::FileNode& fn) -{ - params.read(fn); -} + TrackerGOTURN::Params params; -void TrackerGOTURNImpl::write(cv::FileStorage& fs) const -{ - params.write(fs); -} + dnn::Net net; + Rect boundingBox_; + Mat image_; +}; -bool TrackerGOTURNImpl::initImpl(const Mat& image, const Rect2d& boundingBox) +void TrackerGOTURNImpl::init(InputArray image, const Rect& boundingBox) { - //Make a simple model from frame and bounding box - model = Ptr(new TrackerGOTURNModel(params)); - ((TrackerGOTURNModel*)static_cast(model))->setImage(image); - ((TrackerGOTURNModel*)static_cast(model))->setBoudingBox(boundingBox); - - //Load GOTURN architecture from *.prototxt and pretrained weights from *.caffemodel - net = dnn::readNetFromCaffe(params.modelTxt, params.modelBin); - return true; + image_ = image.getMat().clone(); + setBoudingBox(boundingBox); } -bool TrackerGOTURNImpl::updateImpl(const Mat& image, Rect2d& boundingBox) +bool TrackerGOTURNImpl::update(InputArray image, Rect& boundingBox) { int INPUT_SIZE = 227; //Using prevFrame & prevBB from model and curFrame GOTURN calculating curBB - Mat curFrame = image.clone(); - Mat prevFrame = ((TrackerGOTURNModel*)static_cast(model))->getImage(); - Rect2d prevBB = ((TrackerGOTURNModel*)static_cast(model))->getBoundingBox(); - Rect2d curBB; + InputArray curFrame = image; + Mat prevFrame = image_; + Rect2d prevBB = boundingBox_; + Rect curBB; float padTargetPatch = 2.0; Rect2f searchPatchRect, targetPatchRect; @@ -154,12 +133,12 @@ bool TrackerGOTURNImpl::updateImpl(const Mat& image, Rect2d& boundingBox) copyMakeBorder(curFrame, curFramePadded, (int)targetPatchRect.height, (int)targetPatchRect.height, (int)targetPatchRect.width, (int)targetPatchRect.width, BORDER_REPLICATE); searchPatch = curFramePadded(targetPatchRect).clone(); - //Preprocess - //Resize + // Preprocess + // Resize resize(targetPatch, targetPatch, Size(INPUT_SIZE, INPUT_SIZE), 0, 0, INTER_LINEAR_EXACT); resize(searchPatch, searchPatch, Size(INPUT_SIZE, INPUT_SIZE), 0, 0, INTER_LINEAR_EXACT); - //Convert to Float type and subtract mean + // Convert to Float type and subtract mean Mat targetBlob = dnn::blobFromImage(targetPatch, 1.0f, Size(), Scalar::all(128), false); Mat searchBlob = dnn::blobFromImage(searchPatch, 1.0f, Size(), Scalar::all(128), false); @@ -168,22 +147,31 @@ bool TrackerGOTURNImpl::updateImpl(const Mat& image, Rect2d& boundingBox) Mat resMat = net.forward("scale").reshape(1, 1); - curBB.x = targetPatchRect.x + (resMat.at(0) * targetPatchRect.width / INPUT_SIZE) - targetPatchRect.width; - curBB.y = targetPatchRect.y + (resMat.at(1) * targetPatchRect.height / INPUT_SIZE) - targetPatchRect.height; - curBB.width = (resMat.at(2) - resMat.at(0)) * targetPatchRect.width / INPUT_SIZE; - curBB.height = (resMat.at(3) - resMat.at(1)) * targetPatchRect.height / INPUT_SIZE; + curBB.x = cvRound(targetPatchRect.x + (resMat.at(0) * targetPatchRect.width / INPUT_SIZE) - targetPatchRect.width); + curBB.y = cvRound(targetPatchRect.y + (resMat.at(1) * targetPatchRect.height / INPUT_SIZE) - targetPatchRect.height); + curBB.width = cvRound((resMat.at(2) - resMat.at(0)) * targetPatchRect.width / INPUT_SIZE); + curBB.height = cvRound((resMat.at(3) - resMat.at(1)) * targetPatchRect.height / INPUT_SIZE); - //Predicted BB - boundingBox = curBB; - - //Set new model image and BB from current frame - ((TrackerGOTURNModel*)static_cast(model))->setImage(curFrame); - ((TrackerGOTURNModel*)static_cast(model))->setBoudingBox(curBB); + // Predicted BB + boundingBox = curBB & Rect(Point(0, 0), image_.size()); + // Set new model image and BB from current frame + image_ = image.getMat().clone(); + setBoudingBox(curBB); return true; } +Ptr TrackerGOTURN::create(const TrackerGOTURN::Params& parameters) +{ + return makePtr(parameters); +} + +#else // OPENCV_HAVE_DNN +Ptr TrackerGOTURN::create(const TrackerGOTURN::Params& parameters) +{ + (void)(parameters); + CV_Error(cv::Error::StsNotImplemented, "to use GOTURN, the tracking module needs to be built with opencv_dnn !"); } #endif // OPENCV_HAVE_DNN -} +}} // namespace diff --git a/modules/tracking/src/gtrTracker.hpp b/modules/tracking/src/gtrTracker.hpp deleted file mode 100644 index cd2820da5..000000000 --- a/modules/tracking/src/gtrTracker.hpp +++ /dev/null @@ -1,80 +0,0 @@ - -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_GOTURN_TRACKER -#define OPENCV_GOTURN_TRACKER - -#include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "gtrUtils.hpp" -#include "opencv2/imgproc.hpp" - -#include -#include - -#include "opencv2/opencv_modules.hpp" -#ifdef HAVE_OPENCV_DNN -#include "opencv2/dnn.hpp" - -namespace cv -{ -namespace gtr -{ - -class TrackerGOTURNImpl : public TrackerGOTURN -{ -public: - TrackerGOTURNImpl(const TrackerGOTURN::Params ¶meters = TrackerGOTURN::Params()); - void read(const FileNode& fn) CV_OVERRIDE; - void write(FileStorage& fs) const CV_OVERRIDE; - bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE; - bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE; - - TrackerGOTURN::Params params; - - dnn::Net net; -}; - -} -} -#endif -#endif diff --git a/modules/tracking/src/gtrUtils.cpp b/modules/tracking/src/gtrUtils.cpp index e80dda1ea..d2be7588d 100644 --- a/modules/tracking/src/gtrUtils.cpp +++ b/modules/tracking/src/gtrUtils.cpp @@ -39,6 +39,7 @@ // //M*/ +#include "precomp.hpp" #include "gtrUtils.hpp" diff --git a/modules/tracking/src/gtrUtils.hpp b/modules/tracking/src/gtrUtils.hpp index fedf26f88..0a3dea5ad 100644 --- a/modules/tracking/src/gtrUtils.hpp +++ b/modules/tracking/src/gtrUtils.hpp @@ -1,7 +1,6 @@ #ifndef OPENCV_GTR_UTILS #define OPENCV_GTR_UTILS -#include "precomp.hpp" #include namespace cv diff --git a/modules/tracking/src/kuhn_munkres.cpp b/modules/tracking/src/kuhn_munkres.cpp index 1d33c70e4..5d0acee85 100644 --- a/modules/tracking/src/kuhn_munkres.cpp +++ b/modules/tracking/src/kuhn_munkres.cpp @@ -2,12 +2,17 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" #include "kuhn_munkres.hpp" #include #include #include +namespace cv { +namespace detail { +inline namespace tracking { + KuhnMunkres::KuhnMunkres() : n_() {} std::vector KuhnMunkres::Solve(const cv::Mat& dissimilarity_matrix) { @@ -166,3 +171,6 @@ void KuhnMunkres::Run() { } } } + + +}}} // namespace \ No newline at end of file diff --git a/modules/tracking/src/kuhn_munkres.hpp b/modules/tracking/src/kuhn_munkres.hpp index 4f5ea28c1..d093d33af 100644 --- a/modules/tracking/src/kuhn_munkres.hpp +++ b/modules/tracking/src/kuhn_munkres.hpp @@ -10,6 +10,9 @@ #include #include +namespace cv { +namespace detail { +inline namespace tracking { /// /// \brief The KuhnMunkres class @@ -52,4 +55,6 @@ private: int FindInCol(int col, int what); void Run(); }; + +}}} // namespace #endif // #ifndef __OPENCV_TRACKING_KUHN_MUNKRES_HPP__ diff --git a/modules/tracking/src/legacy/tracker.legacy.hpp b/modules/tracking/src/legacy/tracker.legacy.hpp index f692b5d62..788a758bc 100644 --- a/modules/tracking/src/legacy/tracker.legacy.hpp +++ b/modules/tracking/src/legacy/tracker.legacy.hpp @@ -39,14 +39,16 @@ // //M*/ -#include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" -namespace cv -{ +namespace cv { +namespace legacy { +inline namespace tracking { -/* - * Tracker - */ +Tracker::Tracker() +{ + isInit = false; +} Tracker::~Tracker() { @@ -97,4 +99,42 @@ bool Tracker::update( InputArray image, Rect2d& boundingBox ) return updateImpl( image.getMat(), boundingBox ); } -} /* namespace cv */ + + +class LegacyTrackerWrapper : public cv::tracking::Tracker +{ + const Ptr legacy_tracker_; +public: + LegacyTrackerWrapper(const Ptr& legacy_tracker) : legacy_tracker_(legacy_tracker) + { + CV_Assert(legacy_tracker_); + } + virtual ~LegacyTrackerWrapper() CV_OVERRIDE {}; + + void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE + { + CV_DbgAssert(legacy_tracker_); + legacy_tracker_->init(image, (Rect2d)boundingBox); + } + + bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE + { + CV_DbgAssert(legacy_tracker_); + Rect2d boundingBox2d; + bool res = legacy_tracker_->update(image, boundingBox2d); + int x1 = cvRound(boundingBox2d.x); + int y1 = cvRound(boundingBox2d.y); + int x2 = cvRound(boundingBox2d.x + boundingBox2d.width); + int y2 = cvRound(boundingBox2d.y + boundingBox2d.height); + boundingBox = Rect(x1, y1, x2 - x1, y2 - y1) & Rect(Point(0, 0), image.size()); + return res; + } +}; + + +CV_EXPORTS_W Ptr upgradeTrackingAPI(const Ptr& legacy_tracker) +{ + return makePtr(legacy_tracker); +} + +}}} // namespace diff --git a/modules/tracking/src/legacy/trackerCSRT.legacy.hpp b/modules/tracking/src/legacy/trackerCSRT.legacy.hpp index 900fea51a..b332ef25c 100644 --- a/modules/tracking/src/legacy/trackerCSRT.legacy.hpp +++ b/modules/tracking/src/legacy/trackerCSRT.legacy.hpp @@ -2,657 +2,59 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -#include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" -#include "trackerCSRTSegmentation.hpp" -#include "trackerCSRTUtils.hpp" -#include "trackerCSRTScaleEstimation.hpp" +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { -namespace cv -{ -/** -* \brief Implementation of TrackerModel for CSRT algorithm -*/ -class TrackerCSRTModel : public TrackerModel +class TrackerCSRTImpl CV_FINAL : public legacy::TrackerCSRT { public: - TrackerCSRTModel(TrackerCSRT::Params /*params*/){} - ~TrackerCSRTModel(){} -protected: - void modelEstimationImpl(const std::vector& /*responses*/) CV_OVERRIDE {} - void modelUpdateImpl() CV_OVERRIDE {} -}; - - -class TrackerCSRTImpl : public TrackerCSRT -{ -public: - TrackerCSRTImpl(const TrackerCSRT::Params ¶meters = TrackerCSRT::Params()); - void read(const FileNode& fn) CV_OVERRIDE; - void write(FileStorage& fs) const CV_OVERRIDE; - -protected: - TrackerCSRT::Params params; - - bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE; - virtual void setInitialMask(InputArray mask) CV_OVERRIDE; - bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE; - void update_csr_filter(const Mat &image, const Mat &my_mask); - void update_histograms(const Mat &image, const Rect ®ion); - void extract_histograms(const Mat &image, cv::Rect region, Histogram &hf, Histogram &hb); - std::vector create_csr_filter(const std::vector - img_features, const cv::Mat Y, const cv::Mat P); - Mat calculate_response(const Mat &image, const std::vector filter); - Mat get_location_prior(const Rect roi, const Size2f target_size, const Size img_sz); - Mat segment_region(const Mat &image, const Point2f &object_center, - const Size2f &template_size, const Size &target_size, float scale_factor); - Point2f estimate_new_position(const Mat &image); - std::vector get_features(const Mat &patch, const Size2i &feature_size); - -private: - bool check_mask_area(const Mat &mat, const double obj_area); - float current_scale_factor; - Mat window; - Mat yf; - Rect2f bounding_box; - std::vector csr_filter; - std::vector filter_weights; - Size2f original_target_size; - Size2i image_size; - Size2f template_size; - Size2i rescaled_template_size; - float rescale_ratio; - Point2f object_center; - DSST dsst; - Histogram hist_foreground; - Histogram hist_background; - double p_b; - Mat erode_element; - Mat filter_mask; - Mat preset_mask; - Mat default_mask; - float default_mask_area; - int cell_size; -}; - -Ptr TrackerCSRT::create(const TrackerCSRT::Params ¶meters) -{ - return Ptr(new TrackerCSRTImpl(parameters)); -} -Ptr TrackerCSRT::create() -{ - return Ptr(new TrackerCSRTImpl()); -} -TrackerCSRTImpl::TrackerCSRTImpl(const TrackerCSRT::Params ¶meters) : - params(parameters) -{ - isInit = false; -} - -void TrackerCSRTImpl::read(const cv::FileNode& fn) -{ - params.read(fn); -} - -void TrackerCSRTImpl::write(cv::FileStorage& fs) const -{ - params.write(fs); -} - -void TrackerCSRTImpl::setInitialMask(InputArray mask) -{ - preset_mask = mask.getMat(); -} - -bool TrackerCSRTImpl::check_mask_area(const Mat &mat, const double obj_area) -{ - double threshold = 0.05; - double mask_area= sum(mat)[0]; - if(mask_area < threshold*obj_area) { - return false; - } - return true; -} - -Mat TrackerCSRTImpl::calculate_response(const Mat &image, const std::vector filter) -{ - Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), - cvFloor(current_scale_factor * template_size.height)); - resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); - - std::vector ftrs = get_features(patch, yf.size()); - std::vector Ffeatures = fourier_transform_features(ftrs); - Mat resp, res; - if(params.use_channel_weights){ - res = Mat::zeros(Ffeatures[0].size(), CV_32FC2); - Mat resp_ch; - Mat mul_mat; - for(size_t i = 0; i < Ffeatures.size(); ++i) { - mulSpectrums(Ffeatures[i], filter[i], resp_ch, 0, true); - res += (resp_ch * filter_weights[i]); - } - idft(res, res, DFT_SCALE | DFT_REAL_OUTPUT); - } else { - res = Mat::zeros(Ffeatures[0].size(), CV_32FC2); - Mat resp_ch; - for(size_t i = 0; i < Ffeatures.size(); ++i) { - mulSpectrums(Ffeatures[i], filter[i], resp_ch, 0 , true); - res = res + resp_ch; - } - idft(res, res, DFT_SCALE | DFT_REAL_OUTPUT); - } - return res; -} - -void TrackerCSRTImpl::update_csr_filter(const Mat &image, const Mat &mask) -{ - Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), - cvFloor(current_scale_factor * template_size.height)); - resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); + cv::tracking::impl::TrackerCSRTImpl impl; - std::vector ftrs = get_features(patch, yf.size()); - std::vector Fftrs = fourier_transform_features(ftrs); - std::vector new_csr_filter = create_csr_filter(Fftrs, yf, mask); - //calculate per channel weights - if(params.use_channel_weights) { - Mat current_resp; - double max_val; - float sum_weights = 0; - std::vector new_filter_weights = std::vector(new_csr_filter.size()); - for(size_t i = 0; i < new_csr_filter.size(); ++i) { - mulSpectrums(Fftrs[i], new_csr_filter[i], current_resp, 0, true); - idft(current_resp, current_resp, DFT_SCALE | DFT_REAL_OUTPUT); - minMaxLoc(current_resp, NULL, &max_val, NULL, NULL); - sum_weights += static_cast(max_val); - new_filter_weights[i] = static_cast(max_val); - } - //update filter weights with new values - float updated_sum = 0; - for(size_t i = 0; i < filter_weights.size(); ++i) { - filter_weights[i] = filter_weights[i]*(1.0f - params.weights_lr) + - params.weights_lr * (new_filter_weights[i] / sum_weights); - updated_sum += filter_weights[i]; - } - //normalize weights - for(size_t i = 0; i < filter_weights.size(); ++i) { - filter_weights[i] /= updated_sum; - } - } - for(size_t i = 0; i < csr_filter.size(); ++i) { - csr_filter[i] = (1.0f - params.filter_lr)*csr_filter[i] + params.filter_lr * new_csr_filter[i]; + TrackerCSRTImpl(const legacy::TrackerCSRT::Params ¶meters) + : impl(parameters) + { + isInit = false; } - std::vector().swap(ftrs); - std::vector().swap(Fftrs); -} - -std::vector TrackerCSRTImpl::get_features(const Mat &patch, const Size2i &feature_size) -{ - std::vector features; - if (params.use_hog) { - std::vector hog = get_features_hog(patch, cell_size); - features.insert(features.end(), hog.begin(), - hog.begin()+params.num_hog_channels_used); - } - if (params.use_color_names) { - std::vector cn; - cn = get_features_cn(patch, feature_size); - features.insert(features.end(), cn.begin(), cn.end()); - } - if(params.use_gray) { - Mat gray_m; - cvtColor(patch, gray_m, COLOR_BGR2GRAY); - resize(gray_m, gray_m, feature_size, 0, 0, INTER_CUBIC); - gray_m.convertTo(gray_m, CV_32FC1, 1.0/255.0, -0.5); - features.push_back(gray_m); - } - if(params.use_rgb) { - std::vector rgb_features = get_features_rgb(patch, feature_size); - features.insert(features.end(), rgb_features.begin(), rgb_features.end()); + void read(const FileNode& fn) CV_OVERRIDE + { + static_cast(impl.params).read(fn); } - - for (size_t i = 0; i < features.size(); ++i) { - features.at(i) = features.at(i).mul(window); + void write(FileStorage& fs) const CV_OVERRIDE + { + static_cast(impl.params).write(fs); } - return features; -} -class ParallelCreateCSRFilter : public ParallelLoopBody { -public: - ParallelCreateCSRFilter( - const std::vector img_features, - const cv::Mat Y, - const cv::Mat P, - int admm_iterations, - std::vector &result_filter_): - result_filter(result_filter_) + bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE { - this->img_features = img_features; - this->Y = Y; - this->P = P; - this->admm_iterations = admm_iterations; - } - virtual void operator ()(const Range& range) const CV_OVERRIDE + impl.init(image, boundingBox); + model = impl.model; + sampler = makePtr(); + featureSet = makePtr(); + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE { - for (int i = range.start; i < range.end; i++) { - float mu = 5.0f; - float beta = 3.0f; - float mu_max = 20.0f; - float lambda = mu / 100.0f; - - Mat F = img_features[i]; - - Mat Sxy, Sxx; - mulSpectrums(F, Y, Sxy, 0, true); - mulSpectrums(F, F, Sxx, 0, true); - - Mat H; - H = divide_complex_matrices(Sxy, (Sxx + lambda)); - idft(H, H, DFT_SCALE|DFT_REAL_OUTPUT); - H = H.mul(P); - dft(H, H, DFT_COMPLEX_OUTPUT); - Mat L = Mat::zeros(H.size(), H.type()); //Lagrangian multiplier - Mat G; - for(int iteration = 0; iteration < admm_iterations; ++iteration) { - G = divide_complex_matrices((Sxy + (mu * H) - L) , (Sxx + mu)); - idft((mu * G) + L, H, DFT_SCALE | DFT_REAL_OUTPUT); - float lm = 1.0f / (lambda+mu); - H = H.mul(P*lm); - dft(H, H, DFT_COMPLEX_OUTPUT); - - //Update variables for next iteration - L = L + mu * (G - H); - mu = min(mu_max, beta*mu); - } - result_filter[i] = H; - } + Rect bb; + bool res = impl.update(image, bb); + boundingBox = bb; + return res; } - ParallelCreateCSRFilter& operator=(const ParallelCreateCSRFilter &) { - return *this; + virtual void setInitialMask(InputArray mask) CV_OVERRIDE + { + impl.setInitialMask(mask); } - -private: - int admm_iterations; - Mat Y; - Mat P; - std::vector img_features; - std::vector &result_filter; }; +} // namespace -std::vector TrackerCSRTImpl::create_csr_filter( - const std::vector img_features, - const cv::Mat Y, - const cv::Mat P) -{ - std::vector result_filter; - result_filter.resize(img_features.size()); - ParallelCreateCSRFilter parallelCreateCSRFilter(img_features, Y, P, - params.admm_iterations, result_filter); - parallel_for_(Range(0, static_cast(result_filter.size())), parallelCreateCSRFilter); - - return result_filter; -} - -Mat TrackerCSRTImpl::get_location_prior( - const Rect roi, - const Size2f target_size, - const Size img_sz) -{ - int x1 = cvRound(max(min(roi.x-1, img_sz.width-1) , 0)); - int y1 = cvRound(max(min(roi.y-1, img_sz.height-1) , 0)); - - int x2 = cvRound(min(max(roi.width-1, 0) , img_sz.width-1)); - int y2 = cvRound(min(max(roi.height-1, 0) , img_sz.height-1)); - - Size target_sz; - target_sz.width = target_sz.height = cvFloor(min(target_size.width, target_size.height)); - - double cx = x1 + (x2-x1)/2.; - double cy = y1 + (y2-y1)/2.; - double kernel_size_width = 1.0/(0.5*static_cast(target_sz.width)*1.4142+1); - double kernel_size_height = 1.0/(0.5*static_cast(target_sz.height)*1.4142+1); - - cv::Mat kernel_weight = Mat::zeros(1 + cvFloor(y2 - y1) , 1+cvFloor(-(x1-cx) + (x2-cx)), CV_64FC1); - for (int y = y1; y < y2+1; ++y){ - double * weightPtr = kernel_weight.ptr(y); - double tmp_y = std::pow((cy-y)*kernel_size_height, 2); - for (int x = x1; x < x2+1; ++x){ - weightPtr[x] = kernel_epan(std::pow((cx-x)*kernel_size_width,2) + tmp_y); - } - } - - double max_val; - cv::minMaxLoc(kernel_weight, NULL, &max_val, NULL, NULL); - Mat fg_prior = kernel_weight / max_val; - fg_prior.setTo(0.5, fg_prior < 0.5); - fg_prior.setTo(0.9, fg_prior > 0.9); - return fg_prior; -} - -Mat TrackerCSRTImpl::segment_region( - const Mat &image, - const Point2f &object_center, - const Size2f &template_size, - const Size &target_size, - float scale_factor) -{ - Rect valid_pixels; - Mat patch = get_subwindow(image, object_center, cvFloor(scale_factor * template_size.width), - cvFloor(scale_factor * template_size.height), &valid_pixels); - Size2f scaled_target = Size2f(target_size.width * scale_factor, - target_size.height * scale_factor); - Mat fg_prior = get_location_prior( - Rect(0,0, patch.size().width, patch.size().height), - scaled_target , patch.size()); - - std::vector img_channels; - split(patch, img_channels); - std::pair probs = Segment::computePosteriors2(img_channels, 0, 0, patch.cols, patch.rows, - p_b, fg_prior, 1.0-fg_prior, hist_foreground, hist_background); - - Mat mask = Mat::zeros(probs.first.size(), probs.first.type()); - probs.first(valid_pixels).copyTo(mask(valid_pixels)); - double max_resp = get_max(mask); - threshold(mask, mask, max_resp / 2.0, 1, THRESH_BINARY); - mask.convertTo(mask, CV_32FC1, 1.0); - return mask; -} - - -void TrackerCSRTImpl::extract_histograms(const Mat &image, cv::Rect region, Histogram &hf, Histogram &hb) -{ - // get coordinates of the region - int x1 = std::min(std::max(0, region.x), image.cols-1); - int y1 = std::min(std::max(0, region.y), image.rows-1); - int x2 = std::min(std::max(0, region.x + region.width), image.cols-1); - int y2 = std::min(std::max(0, region.y + region.height), image.rows-1); - - // calculate coordinates of the background region - int offsetX = (x2-x1+1) / params.background_ratio; - int offsetY = (y2-y1+1) / params.background_ratio; - int outer_y1 = std::max(0, (int)(y1-offsetY)); - int outer_y2 = std::min(image.rows, (int)(y2+offsetY+1)); - int outer_x1 = std::max(0, (int)(x1-offsetX)); - int outer_x2 = std::min(image.cols, (int)(x2+offsetX+1)); - - // calculate probability for the background - p_b = 1.0 - ((x2-x1+1) * (y2-y1+1)) / - ((double) (outer_x2-outer_x1+1) * (outer_y2-outer_y1+1)); - - // split multi-channel image into the std::vector of matrices - std::vector img_channels(image.channels()); - split(image, img_channels); - for(size_t k=0; k().swap(img_channels); -} - -void TrackerCSRTImpl::update_histograms(const Mat &image, const Rect ®ion) -{ - // create temporary histograms - Histogram hf(image.channels(), params.histogram_bins); - Histogram hb(image.channels(), params.histogram_bins); - extract_histograms(image, region, hf, hb); - - // get histogram vectors from temporary histograms - std::vector hf_vect_new = hf.getHistogramVector(); - std::vector hb_vect_new = hb.getHistogramVector(); - // get histogram vectors from learned histograms - std::vector hf_vect = hist_foreground.getHistogramVector(); - std::vector hb_vect = hist_background.getHistogramVector(); - - // update histograms - use learning rate - for(size_t i=0; i().swap(hf_vect); - std::vector().swap(hb_vect); -} - -Point2f TrackerCSRTImpl::estimate_new_position(const Mat &image) -{ - - Mat resp = calculate_response(image, csr_filter); - - double max_val; - Point max_loc; - minMaxLoc(resp, NULL, &max_val, NULL, &max_loc); - if (max_val < params.psr_threshold) - return Point2f(-1,-1); // target "lost" - - // take into account also subpixel accuracy - float col = ((float) max_loc.x) + subpixel_peak(resp, "horizontal", max_loc); - float row = ((float) max_loc.y) + subpixel_peak(resp, "vertical", max_loc); - if(row + 1 > (float)resp.rows / 2.0f) { - row = row - resp.rows; - } - if(col + 1 > (float)resp.cols / 2.0f) { - col = col - resp.cols; - } - // calculate x and y displacements - Point2f new_center = object_center + Point2f(current_scale_factor * (1.0f / rescale_ratio) *cell_size*(col), - current_scale_factor * (1.0f / rescale_ratio) *cell_size*(row)); - //sanity checks - if(new_center.x < 0) - new_center.x = 0; - if(new_center.x >= image_size.width) - new_center.x = static_cast(image_size.width - 1); - if(new_center.y < 0) - new_center.y = 0; - if(new_center.y >= image_size.height) - new_center.y = static_cast(image_size.height - 1); - - return new_center; -} - -// ********************************************************************* -// * Update API function * -// ********************************************************************* -bool TrackerCSRTImpl::updateImpl(const Mat& image_, Rect2d& boundingBox) -{ - Mat image; - if(image_.channels() == 1) //treat gray image as color image - cvtColor(image_, image, COLOR_GRAY2BGR); - else - image = image_; - - object_center = estimate_new_position(image); - if (object_center.x < 0 && object_center.y < 0) - return false; - - current_scale_factor = dsst.getScale(image, object_center); - //update bouding_box according to new scale and location - bounding_box.x = object_center.x - current_scale_factor * original_target_size.width / 2.0f; - bounding_box.y = object_center.y - current_scale_factor * original_target_size.height / 2.0f; - bounding_box.width = current_scale_factor * original_target_size.width; - bounding_box.height = current_scale_factor * original_target_size.height; - - //update tracker - if(params.use_segmentation) { - Mat hsv_img = bgr2hsv(image); - update_histograms(hsv_img, bounding_box); - filter_mask = segment_region(hsv_img, object_center, - template_size,original_target_size, current_scale_factor); - resize(filter_mask, filter_mask, yf.size(), 0, 0, INTER_NEAREST); - if(check_mask_area(filter_mask, default_mask_area)) { - dilate(filter_mask , filter_mask, erode_element); - } else { - filter_mask = default_mask; - } - } else { - filter_mask = default_mask; - } - update_csr_filter(image, filter_mask); - dsst.update(image, object_center); - boundingBox = bounding_box; - return true; -} - -// ********************************************************************* -// * Init API function * -// ********************************************************************* -bool TrackerCSRTImpl::initImpl(const Mat& image_, const Rect2d& boundingBox) -{ - Mat image; - if(image_.channels() == 1) //treat gray image as color image - cvtColor(image_, image, COLOR_GRAY2BGR); - else - image = image_; - - current_scale_factor = 1.0; - image_size = image.size(); - bounding_box = boundingBox; - cell_size = cvFloor(std::min(4.0, std::max(1.0, static_cast( - cvCeil((bounding_box.width * bounding_box.height)/400.0))))); - original_target_size = Size(bounding_box.size()); - - template_size.width = static_cast(cvFloor(original_target_size.width + params.padding * - sqrt(original_target_size.width * original_target_size.height))); - template_size.height = static_cast(cvFloor(original_target_size.height + params.padding * - sqrt(original_target_size.width * original_target_size.height))); - template_size.width = template_size.height = - (template_size.width + template_size.height) / 2.0f; - rescale_ratio = sqrt(pow(params.template_size,2) / (template_size.width * template_size.height)); - if(rescale_ratio > 1) { - rescale_ratio = 1; - } - rescaled_template_size = Size2i(cvFloor(template_size.width * rescale_ratio), - cvFloor(template_size.height * rescale_ratio)); - object_center = Point2f(static_cast(boundingBox.x) + original_target_size.width / 2.0f, - static_cast(boundingBox.y) + original_target_size.height / 2.0f); - - yf = gaussian_shaped_labels(params.gsl_sigma, - rescaled_template_size.width / cell_size, rescaled_template_size.height / cell_size); - if(params.window_function.compare("hann") == 0) { - window = get_hann_win(Size(yf.cols,yf.rows)); - } else if(params.window_function.compare("cheb") == 0) { - window = get_chebyshev_win(Size(yf.cols,yf.rows), params.cheb_attenuation); - } else if(params.window_function.compare("kaiser") == 0) { - window = get_kaiser_win(Size(yf.cols,yf.rows), params.kaiser_alpha); - } else { - std::cout << "Not a valid window function" << std::endl; - return false; - } - - Size2i scaled_obj_size = Size2i(cvFloor(original_target_size.width * rescale_ratio / cell_size), - cvFloor(original_target_size.height * rescale_ratio / cell_size)); - //set dummy mask and area; - int x0 = std::max((yf.size().width - scaled_obj_size.width)/2 - 1, 0); - int y0 = std::max((yf.size().height - scaled_obj_size.height)/2 - 1, 0); - default_mask = Mat::zeros(yf.size(), CV_32FC1); - default_mask(Rect(x0,y0,scaled_obj_size.width, scaled_obj_size.height)) = 1.0f; - default_mask_area = static_cast(sum(default_mask)[0]); - - //initalize segmentation - if(params.use_segmentation) { - Mat hsv_img = bgr2hsv(image); - hist_foreground = Histogram(hsv_img.channels(), params.histogram_bins); - hist_background = Histogram(hsv_img.channels(), params.histogram_bins); - extract_histograms(hsv_img, bounding_box, hist_foreground, hist_background); - filter_mask = segment_region(hsv_img, object_center, template_size, - original_target_size, current_scale_factor); - //update calculated mask with preset mask - if(preset_mask.data){ - Mat preset_mask_padded = Mat::zeros(filter_mask.size(), filter_mask.type()); - int sx = std::max((int)cvFloor(preset_mask_padded.cols / 2.0f - preset_mask.cols / 2.0f) - 1, 0); - int sy = std::max((int)cvFloor(preset_mask_padded.rows / 2.0f - preset_mask.rows / 2.0f) - 1, 0); - preset_mask.copyTo(preset_mask_padded( - Rect(sx, sy, preset_mask.cols, preset_mask.rows))); - filter_mask = filter_mask.mul(preset_mask_padded); - } - erode_element = getStructuringElement(MORPH_ELLIPSE, Size(3,3), Point(1,1)); - resize(filter_mask, filter_mask, yf.size(), 0, 0, INTER_NEAREST); - if(check_mask_area(filter_mask, default_mask_area)) { - dilate(filter_mask , filter_mask, erode_element); - } else { - filter_mask = default_mask; - } - - } else { - filter_mask = default_mask; - } - - //initialize filter - Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), - cvFloor(current_scale_factor * template_size.height)); - resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); - std::vector patch_ftrs = get_features(patch, yf.size()); - std::vector Fftrs = fourier_transform_features(patch_ftrs); - csr_filter = create_csr_filter(Fftrs, yf, filter_mask); - - if(params.use_channel_weights) { - Mat current_resp; - filter_weights = std::vector(csr_filter.size()); - float chw_sum = 0; - for (size_t i = 0; i < csr_filter.size(); ++i) { - mulSpectrums(Fftrs[i], csr_filter[i], current_resp, 0, true); - idft(current_resp, current_resp, DFT_SCALE | DFT_REAL_OUTPUT); - double max_val; - minMaxLoc(current_resp, NULL, &max_val, NULL , NULL); - chw_sum += static_cast(max_val); - filter_weights[i] = static_cast(max_val); - } - for (size_t i = 0; i < filter_weights.size(); ++i) { - filter_weights[i] /= chw_sum; - } - } - - //initialize scale search - dsst = DSST(image, bounding_box, template_size, params.number_of_scales, params.scale_step, - params.scale_model_max_area, params.scale_sigma_factor, params.scale_lr); - - model=Ptr(new TrackerCSRTModel(params)); - isInit = true; - return true; -} - -TrackerCSRT::Params::Params() -{ - use_channel_weights = true; - use_segmentation = true; - use_hog = true; - use_color_names = true; - use_gray = true; - use_rgb = false; - window_function = "hann"; - kaiser_alpha = 3.75f; - cheb_attenuation = 45; - padding = 3.0f; - template_size = 200; - gsl_sigma = 1.0f; - hog_orientations = 9; - hog_clip = 0.2f; - num_hog_channels_used = 18; - filter_lr = 0.02f; - weights_lr = 0.02f; - admm_iterations = 4; - number_of_scales = 33; - scale_sigma_factor = 0.250f; - scale_model_max_area = 512.0f; - scale_lr = 0.025f; - scale_step = 1.020f; - histogram_bins = 16; - background_ratio = 2; - histogram_lr = 0.04f; - psr_threshold = 0.035f; -} - -void TrackerCSRT::Params::read(const FileNode& fn) +void legacy::TrackerCSRT::Params::read(const FileNode& fn) { *this = TrackerCSRT::Params(); if(!fn["padding"].empty()) @@ -712,7 +114,7 @@ void TrackerCSRT::Params::read(const FileNode& fn) CV_Assert(number_of_scales % 2 == 1); CV_Assert(use_gray || use_color_names || use_hog || use_rgb); } -void TrackerCSRT::Params::write(FileStorage& fs) const +void legacy::TrackerCSRT::Params::write(FileStorage& fs) const { fs << "padding" << padding; fs << "template_size" << template_size; @@ -742,4 +144,16 @@ void TrackerCSRT::Params::write(FileStorage& fs) const fs << "histogram_lr" << histogram_lr; fs << "psr_threshold" << psr_threshold; } -} /* namespace cv */ + +}} // namespace + +Ptr legacy::TrackerCSRT::create(const legacy::TrackerCSRT::Params ¶meters) +{ + return makePtr(parameters); +} +Ptr legacy::TrackerCSRT::create() +{ + return create(legacy::TrackerCSRT::Params()); +} + +} // namespace diff --git a/modules/tracking/src/legacy/trackerKCF.legacy.hpp b/modules/tracking/src/legacy/trackerKCF.legacy.hpp index 471ad71f7..86e895ec2 100644 --- a/modules/tracking/src/legacy/trackerKCF.legacy.hpp +++ b/modules/tracking/src/legacy/trackerKCF.legacy.hpp @@ -39,893 +39,61 @@ // //M*/ -#include "precomp.hpp" -#include "opencl_kernels_tracking.hpp" -#include -#include - -/*--------------------------- -| TrackerKCFModel -|---------------------------*/ -namespace cv{ - /** - * \brief Implementation of TrackerModel for KCF algorithm - */ - class TrackerKCFModel : public TrackerModel{ - public: - TrackerKCFModel(TrackerKCF::Params /*params*/){} - ~TrackerKCFModel(){} - protected: - void modelEstimationImpl( const std::vector& /*responses*/ ) CV_OVERRIDE {} - void modelUpdateImpl() CV_OVERRIDE {} - }; -} /* namespace cv */ +#include "opencv2/tracking/tracking_legacy.hpp" +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { /*--------------------------- | TrackerKCF |---------------------------*/ -namespace cv{ - - /* - * Prototype - */ - class TrackerKCFImpl : public TrackerKCF { - public: - TrackerKCFImpl( const TrackerKCF::Params ¶meters = TrackerKCF::Params() ); - void read( const FileNode& /*fn*/ ) CV_OVERRIDE; - void write( FileStorage& /*fs*/ ) const CV_OVERRIDE; - void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE; - - protected: - /* - * basic functions and vars - */ - bool initImpl( const Mat& /*image*/, const Rect2d& boundingBox ) CV_OVERRIDE; - bool updateImpl( const Mat& image, Rect2d& boundingBox ) CV_OVERRIDE; - - TrackerKCF::Params params; - - /* - * KCF functions and vars - */ - void createHanningWindow(OutputArray dest, const cv::Size winSize, const int type) const; - void inline fft2(const Mat src, std::vector & dest, std::vector & layers_data) const; - void inline fft2(const Mat src, Mat & dest) const; - void inline ifft2(const Mat src, Mat & dest) const; - void inline pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB=false) const; - void inline sumChannels(std::vector src, Mat & dest) const; - void inline updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & proj_matrix,float pca_rate, int compressed_sz, - std::vector & layers_pca,std::vector & average, Mat pca_data, Mat new_cov, Mat w, Mat u, Mat v); - void inline compress(const Mat proj_matrix, const Mat src, Mat & dest, Mat & data, Mat & compressed) const; - bool getSubWindow(const Mat img, const Rect roi, Mat& feat, Mat& patch, TrackerKCF::MODE desc = GRAY) const; - bool getSubWindow(const Mat img, const Rect roi, Mat& feat, void (*f)(const Mat, const Rect, Mat& )) const; - void extractCN(Mat patch_data, Mat & cnFeatures) const; - void denseGaussKernel(const float sigma, const Mat , const Mat y_data, Mat & k_data, - std::vector & layers_data,std::vector & xf_data,std::vector & yf_data, std::vector xyf_v, Mat xy, Mat xyf ) const; - void calcResponse(const Mat alphaf_data, const Mat kf_data, Mat & response_data, Mat & spec_data) const; - void calcResponse(const Mat alphaf_data, const Mat alphaf_den_data, const Mat kf_data, Mat & response_data, Mat & spec_data, Mat & spec2_data) const; - - void shiftRows(Mat& mat) const; - void shiftRows(Mat& mat, int n) const; - void shiftCols(Mat& mat, int n) const; -#ifdef HAVE_OPENCL - bool inline oclTransposeMM(const Mat src, float alpha, UMat &dst); -#endif - - private: - float output_sigma; - Rect2d roi; - Mat hann; //hann window filter - Mat hann_cn; //10 dimensional hann-window filter for CN features, - - Mat y,yf; // training response and its FFT - Mat x; // observation and its FFT - Mat k,kf; // dense gaussian kernel and its FFT - Mat kf_lambda; // kf+lambda - Mat new_alphaf, alphaf; // training coefficients - Mat new_alphaf_den, alphaf_den; // for splitted training coefficients - Mat z; // model - Mat response; // detection result - Mat old_cov_mtx, proj_mtx; // for feature compression - - // pre-defined Mat variables for optimization of private functions - Mat spec, spec2; - std::vector layers; - std::vector vxf,vyf,vxyf; - Mat xy_data,xyf_data; - Mat data_temp, compress_data; - std::vector layers_pca_data; - std::vector average_data; - Mat img_Patch; +class TrackerKCFImpl CV_FINAL : public legacy::TrackerKCF +{ +public: + cv::tracking::impl::TrackerKCFImpl impl; - // storage for the extracted features, KRLS model, KRLS compressed model - Mat X[2],Z[2],Zc[2]; - - // storage of the extracted features - std::vector features_pca; - std::vector features_npca; - std::vector descriptors_pca; - std::vector descriptors_npca; - - // optimization variables for updateProjectionMatrix - Mat data_pca, new_covar,w_data,u_data,vt_data; - - // custom feature extractor - bool use_custom_extractor_pca; - bool use_custom_extractor_npca; - std::vector extractor_pca; - std::vector extractor_npca; - - bool resizeImage; // resize the image whenever needed and the patch size is large - -#ifdef HAVE_OPENCL - ocl::Kernel transpose_mm_ker; // OCL kernel to compute transpose matrix multiply matrix. -#endif - - int frame; - }; - - /* - * Constructor - */ - Ptr TrackerKCF::create(const TrackerKCF::Params ¶meters){ - return Ptr(new TrackerKCFImpl(parameters)); - } - Ptr TrackerKCF::create(){ - return Ptr(new TrackerKCFImpl()); - } - TrackerKCFImpl::TrackerKCFImpl( const TrackerKCF::Params ¶meters ) : - params( parameters ) - { - isInit = false; - resizeImage = false; - use_custom_extractor_pca = false; - use_custom_extractor_npca = false; - -#ifdef HAVE_OPENCL - // For update proj matrix's multiplication - if(ocl::useOpenCL()) + TrackerKCFImpl(const legacy::TrackerKCF::Params ¶meters) + : impl(parameters) { - cv::String err; - ocl::ProgramSource tmmSrc = ocl::tracking::tmm_oclsrc; - ocl::Program tmmProg(tmmSrc, String(), err); - transpose_mm_ker.create("tmm", tmmProg); - } -#endif - } - - void TrackerKCFImpl::read( const cv::FileNode& fn ){ - params.read( fn ); - } - - void TrackerKCFImpl::write( cv::FileStorage& fs ) const { - params.write( fs ); - } - - /* - * Initialization: - * - creating hann window filter - * - ROI padding - * - creating a gaussian response for the training ground-truth - * - perform FFT to the gaussian response - */ - bool TrackerKCFImpl::initImpl( const Mat& image, const Rect2d& boundingBox ){ - frame=0; - roi.x = cvRound(boundingBox.x); - roi.y = cvRound(boundingBox.y); - roi.width = cvRound(boundingBox.width); - roi.height = cvRound(boundingBox.height); - - //calclulate output sigma - output_sigma=std::sqrt(static_cast(roi.width*roi.height))*params.output_sigma_factor; - output_sigma=-0.5f/(output_sigma*output_sigma); - - //resize the ROI whenever needed - if(params.resize && roi.width*roi.height>params.max_patch_size){ - resizeImage=true; - roi.x/=2.0; - roi.y/=2.0; - roi.width/=2.0; - roi.height/=2.0; - } - - // add padding to the roi - roi.x-=roi.width/2; - roi.y-=roi.height/2; - roi.width*=2; - roi.height*=2; - - // initialize the hann window filter - createHanningWindow(hann, roi.size(), CV_32F); - - // hann window filter for CN feature - Mat _layer[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; - merge(_layer, 10, hann_cn); - - // create gaussian response - y=Mat::zeros((int)roi.height,(int)roi.width,CV_32F); - for(int i=0;i(i,j) = - static_cast((i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1)); - } - } - - y*=(float)output_sigma; - cv::exp(y,y); - - // perform fourier transfor to the gaussian response - fft2(y,yf); - - if (image.channels() == 1) { // disable CN for grayscale images - params.desc_pca &= ~(CN); - params.desc_npca &= ~(CN); - } - model=Ptr(new TrackerKCFModel(params)); - - // record the non-compressed descriptors - if((params.desc_npca & GRAY) == GRAY)descriptors_npca.push_back(GRAY); - if((params.desc_npca & CN) == CN)descriptors_npca.push_back(CN); - if(use_custom_extractor_npca)descriptors_npca.push_back(CUSTOM); - features_npca.resize(descriptors_npca.size()); - - // record the compressed descriptors - if((params.desc_pca & GRAY) == GRAY)descriptors_pca.push_back(GRAY); - if((params.desc_pca & CN) == CN)descriptors_pca.push_back(CN); - if(use_custom_extractor_pca)descriptors_pca.push_back(CUSTOM); - features_pca.resize(descriptors_pca.size()); - - // accept only the available descriptor modes - CV_Assert( - (params.desc_pca & GRAY) == GRAY - || (params.desc_npca & GRAY) == GRAY - || (params.desc_pca & CN) == CN - || (params.desc_npca & CN) == CN - || use_custom_extractor_pca - || use_custom_extractor_npca - ); - - //return true only if roi has intersection with the image - if((roi & Rect2d(0,0, resizeImage ? image.cols / 2 : image.cols, - resizeImage ? image.rows / 2 : image.rows)) == Rect2d()) - return false; - - return true; - } - - /* - * Main part of the KCF algorithm - */ - bool TrackerKCFImpl::updateImpl( const Mat& image, Rect2d& boundingBox ){ - double minVal, maxVal; // min-max response - Point minLoc,maxLoc; // min-max location - - Mat img=image.clone(); - // check the channels of the input image, grayscale is preferred - CV_Assert(img.channels() == 1 || img.channels() == 3); - - // resize the image whenever needed - if(resizeImage)resize(img,img,Size(img.cols/2,img.rows/2),0,0,INTER_LINEAR_EXACT); - - // detection part - if(frame>0){ - - // extract and pre-process the patch - // get non compressed descriptors - for(unsigned i=0;i0)merge(features_npca,X[1]); - - // get compressed descriptors - for(unsigned i=0;i0)merge(features_pca,X[0]); - - //compress the features and the KRSL model - if(params.desc_pca !=0){ - compress(proj_mtx,X[0],X[0],data_temp,compress_data); - compress(proj_mtx,Z[0],Zc[0],data_temp,compress_data); - } - - // copy the compressed KRLS model - Zc[1] = Z[1]; - - // merge all features - if(features_npca.size()==0){ - x = X[0]; - z = Zc[0]; - }else if(features_pca.size()==0){ - x = X[1]; - z = Z[1]; - }else{ - merge(X,2,x); - merge(Zc,2,z); - } - - //compute the gaussian kernel - denseGaussKernel(params.sigma,x,z,k,layers,vxf,vyf,vxyf,xy_data,xyf_data); - - // compute the fourier transform of the kernel - fft2(k,kf); - if(frame==1)spec2=Mat_(kf.rows, kf.cols); - - // calculate filter response - if(params.split_coeff) - calcResponse(alphaf,alphaf_den,kf,response, spec, spec2); - else - calcResponse(alphaf,kf,response, spec); - - // extract the maximum response - minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); - if (maxVal < params.detect_thresh) - { - return false; - } - roi.x+=(maxLoc.x-roi.width/2+1); - roi.y+=(maxLoc.y-roi.height/2+1); - } - - // update the bounding box - boundingBox.x=(resizeImage?roi.x*2:roi.x)+(resizeImage?roi.width*2:roi.width)/4; - boundingBox.y=(resizeImage?roi.y*2:roi.y)+(resizeImage?roi.height*2:roi.height)/4; - boundingBox.width = (resizeImage?roi.width*2:roi.width)/2; - boundingBox.height = (resizeImage?roi.height*2:roi.height)/2; - - // extract the patch for learning purpose - // get non compressed descriptors - for(unsigned i=0;i0)merge(features_npca,X[1]); - - // get compressed descriptors - for(unsigned i=0;i0)merge(features_pca,X[0]); - - //update the training data - if(frame==0){ - Z[0] = X[0].clone(); - Z[1] = X[1].clone(); - }else{ - Z[0]=(1.0-params.interp_factor)*Z[0]+params.interp_factor*X[0]; - Z[1]=(1.0-params.interp_factor)*Z[1]+params.interp_factor*X[1]; - } - - if(params.desc_pca !=0 || use_custom_extractor_pca){ - // initialize the vector of Mat variables - if(frame==0){ - layers_pca_data.resize(Z[0].channels()); - average_data.resize(Z[0].channels()); - } - - // feature compression - updateProjectionMatrix(Z[0],old_cov_mtx,proj_mtx,params.pca_learning_rate,params.compressed_size,layers_pca_data,average_data,data_pca, new_covar,w_data,u_data,vt_data); - compress(proj_mtx,X[0],X[0],data_temp,compress_data); - } - - // merge all features - if(features_npca.size()==0) - x = X[0]; - else if(features_pca.size()==0) - x = X[1]; - else - merge(X,2,x); - - // initialize some required Mat variables - if(frame==0){ - layers.resize(x.channels()); - vxf.resize(x.channels()); - vyf.resize(x.channels()); - vxyf.resize(vyf.size()); - new_alphaf=Mat_(yf.rows, yf.cols); - } - - // Kernel Regularized Least-Squares, calculate alphas - denseGaussKernel(params.sigma,x,x,k,layers,vxf,vyf,vxyf,xy_data,xyf_data); - - // compute the fourier transform of the kernel and add a small value - fft2(k,kf); - kf_lambda=kf+params.lambda; - - float den; - if(params.split_coeff){ - mulSpectrums(yf,kf,new_alphaf,0); - mulSpectrums(kf,kf_lambda,new_alphaf_den,0); - }else{ - for(int i=0;i(i,j)[0]*kf_lambda.at(i,j)[0]+kf_lambda.at(i,j)[1]*kf_lambda.at(i,j)[1]); - - new_alphaf.at(i,j)[0]= - (yf.at(i,j)[0]*kf_lambda.at(i,j)[0]+yf.at(i,j)[1]*kf_lambda.at(i,j)[1])*den; - new_alphaf.at(i,j)[1]= - (yf.at(i,j)[1]*kf_lambda.at(i,j)[0]-yf.at(i,j)[0]*kf_lambda.at(i,j)[1])*den; - } - } - } - - // update the RLS model - if(frame==0){ - alphaf=new_alphaf.clone(); - if(params.split_coeff)alphaf_den=new_alphaf_den.clone(); - }else{ - alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; - if(params.split_coeff)alphaf_den=(1.0-params.interp_factor)*alphaf_den+params.interp_factor*new_alphaf_den; - } - - frame++; - return true; - } - - - /*------------------------------------- - | implementation of the KCF functions - |-------------------------------------*/ - - /* - * hann window filter - */ - void TrackerKCFImpl::createHanningWindow(OutputArray dest, const cv::Size winSize, const int type) const { - CV_Assert( type == CV_32FC1 || type == CV_64FC1 ); - - dest.create(winSize, type); - Mat dst = dest.getMat(); - - int rows = dst.rows, cols = dst.cols; - - AutoBuffer _wc(cols); - float * const wc = _wc.data(); - - const float coeff0 = 2.0f * (float)CV_PI / (cols - 1); - const float coeff1 = 2.0f * (float)CV_PI / (rows - 1); - for(int j = 0; j < cols; j++) - wc[j] = 0.5f * (1.0f - cos(coeff0 * j)); - - if(dst.depth() == CV_32F){ - for(int i = 0; i < rows; i++){ - float* dstData = dst.ptr(i); - float wr = 0.5f * (1.0f - cos(coeff1 * i)); - for(int j = 0; j < cols; j++) - dstData[j] = (float)(wr * wc[j]); - } - }else{ - for(int i = 0; i < rows; i++){ - double* dstData = dst.ptr(i); - double wr = 0.5f * (1.0f - cos(coeff1 * i)); - for(int j = 0; j < cols; j++) - dstData[j] = wr * wc[j]; - } - } - - // perform batch sqrt for SSE performance gains - //cv::sqrt(dst, dst); //matlab do not use the square rooted version - } - - /* - * simplification of fourier transform function in opencv - */ - void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest) const { - dft(src,dest,DFT_COMPLEX_OUTPUT); - } - - void inline TrackerKCFImpl::fft2(const Mat src, std::vector & dest, std::vector & layers_data) const { - split(src, layers_data); - - for(int i=0;i src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const { - for(unsigned i=0;i src, Mat & dest) const { - dest=src[0].clone(); - for(unsigned i=1;i(src.cols * 64), static_cast(src.cols)}; - size_t localSize[2] = {64, 1}; - if (!transpose_mm_ker.run(2, globSize, localSize, true)) - return false; - return true; - } -#endif - - /* - * obtains the projection matrix using PCA - */ - void inline TrackerKCFImpl::updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & proj_matrix, float pca_rate, int compressed_sz, - std::vector & layers_pca,std::vector & average, Mat pca_data, Mat new_cov, Mat w, Mat u, Mat vt) { - CV_Assert(compressed_sz<=src.channels()); - - split(src,layers_pca); - - for (int i=0;i(i, j) - result.getMat(ACCESS_RW).at(i , j)) > abs(new_cov.at(i, j)) * 1e-3) - printf("error @ i %d j %d got %G expected %G \n", i, j, result.getMat(ACCESS_RW).at(i , j), new_cov.at(i, j)); -#endif - if(old_cov.rows==0)old_cov=new_cov.clone(); - SVD::compute((1.0f - pca_rate) * old_cov + pca_rate * new_cov, w, u, vt); - } -#else - new_cov=1.0/(float)(src.rows*src.cols-1)*(pca_data.t()*pca_data); - if(old_cov.rows==0)old_cov=new_cov.clone(); - - // calc PCA - SVD::compute((1.0-pca_rate)*old_cov+pca_rate*new_cov, w, u, vt); -#endif - // extract the projection matrix - proj_matrix=u(Rect(0,0,compressed_sz,src.channels())).clone(); - Mat proj_vars=Mat::eye(compressed_sz,compressed_sz,proj_matrix.type()); - for(int i=0;i(i,i)=w.at(i); + isInit = false; } - - // update the covariance matrix - old_cov=(1.0-pca_rate)*old_cov+pca_rate*proj_matrix*proj_vars*proj_matrix.t(); - } - - /* - * compress the features - */ - void inline TrackerKCFImpl::compress(const Mat proj_matrix, const Mat src, Mat & dest, Mat & data, Mat & compressed) const { - data=src.reshape(1,src.rows*src.cols); - compressed=data*proj_matrix; - dest=compressed.reshape(proj_matrix.cols,src.rows).clone(); - } - - /* - * obtain the patch and apply hann window filter to it - */ - bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& feat, Mat& patch, TrackerKCF::MODE desc) const { - - Rect region=_roi; - - // return false if roi is outside the image - if((roi & Rect2d(0,0, img.cols, img.rows)) == Rect2d() ) - return false; - - // extract patch inside the image - if(_roi.x<0){region.x=0;region.width+=_roi.x;} - if(_roi.y<0){region.y=0;region.height+=_roi.y;} - if(_roi.x+_roi.width>img.cols)region.width=img.cols-_roi.x; - if(_roi.y+_roi.height>img.rows)region.height=img.rows-_roi.y; - if(region.width>img.cols)region.width=img.cols; - if(region.height>img.rows)region.height=img.rows; - - // return false if region is empty - if (region.empty()) - return false; - - patch=img(region).clone(); - - // add some padding to compensate when the patch is outside image border - int addTop,addBottom, addLeft, addRight; - addTop=region.y-_roi.y; - addBottom=(_roi.height+_roi.y>img.rows?_roi.height+_roi.y-img.rows:0); - addLeft=region.x-_roi.x; - addRight=(_roi.width+_roi.x>img.cols?_roi.width+_roi.x-img.cols:0); - - copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); - if(patch.rows==0 || patch.cols==0)return false; - - // extract the desired descriptors - switch(desc){ - case CN: - CV_Assert(img.channels() == 3); - extractCN(patch,feat); - feat=feat.mul(hann_cn); // hann window filter - break; - default: // GRAY - if(img.channels()>1) - cvtColor(patch,feat, COLOR_BGR2GRAY); - else - feat=patch; - //feat.convertTo(feat,CV_32F); - feat.convertTo(feat,CV_32F, 1.0/255.0, -0.5); - //feat=feat/255.0-0.5; // normalize to range -0.5 .. 0.5 - feat=feat.mul(hann); // hann window filter - break; - } - - return true; - - } - - /* - * get feature using external function - */ - bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& feat, void (*f)(const Mat, const Rect, Mat& )) const{ - - // return false if roi is outside the image - if((_roi.x+_roi.width<0) - ||(_roi.y+_roi.height<0) - ||(_roi.x>=img.cols) - ||(_roi.y>=img.rows) - )return false; - - f(img, _roi, feat); - - if(_roi.width != feat.cols || _roi.height != feat.rows){ - printf("error in customized function of features extractor!\n"); - printf("Rules: roi.width==feat.cols && roi.height = feat.rows \n"); - } - - Mat hann_win; - std::vector _layers; - - for(int i=0;i(0,0); - unsigned index; - - if(cnFeatures.type() != CV_32FC(10)) - cnFeatures = Mat::zeros(patch_data.rows,patch_data.cols,CV_32FC(10)); - - for(int i=0;i(i,j); - index=(unsigned)(floor((float)pixel[2]/8)+32*floor((float)pixel[1]/8)+32*32*floor((float)pixel[0]/8)); - - //copy the values - for(int _k=0;_k<10;_k++){ - cnFeatures.at >(i,j)[_k]=ColorNames[index][_k]; - } - } - } - - } - - /* - * dense gauss kernel function - */ - void TrackerKCFImpl::denseGaussKernel(const float sigma, const Mat x_data, const Mat y_data, Mat & k_data, - std::vector & layers_data,std::vector & xf_data,std::vector & yf_data, std::vector xyf_v, Mat xy, Mat xyf ) const { - double normX, normY; - - fft2(x_data,xf_data,layers_data); - fft2(y_data,yf_data,layers_data); - - normX=norm(x_data); - normX*=normX; - normY=norm(y_data); - normY*=normY; - - pixelWiseMult(xf_data,yf_data,xyf_v,0,true); - sumChannels(xyf_v,xyf); - ifft2(xyf,xyf); - - if(params.wrap_kernel){ - shiftRows(xyf, x_data.rows/2); - shiftCols(xyf, x_data.cols/2); + void read(const FileNode& fn) CV_OVERRIDE + { + static_cast(impl.params).read(fn); } - - //(xx + yy - 2 * xy) / numel(x) - xy=(normX+normY-2*xyf)/(x_data.rows*x_data.cols*x_data.channels()); - - // TODO: check wether we really need thresholding or not - //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) - for(int i=0;i(i,j)<0.0)xy.at(i,j)=0.0; - } + void write(FileStorage& fs) const CV_OVERRIDE + { + static_cast(impl.params).write(fs); } - float sig=-1.0f/(sigma*sigma); - xy=sig*xy; - exp(xy,k_data); - - } - - /* CIRCULAR SHIFT Function - * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv - */ - // circular shift one row from up to down - void TrackerKCFImpl::shiftRows(Mat& mat) const { - - Mat temp; - Mat m; - int _k = (mat.rows-1); - mat.row(_k).copyTo(temp); - for(; _k > 0 ; _k-- ) { - m = mat.row(_k); - mat.row(_k-1).copyTo(m); - } - m = mat.row(0); - temp.copyTo(m); - - } - - // circular shift n rows from up to down if n > 0, -n rows from down to up if n < 0 - void TrackerKCFImpl::shiftRows(Mat& mat, int n) const { - if( n < 0 ) { - n = -n; - flip(mat,mat,0); - for(int _k=0; _k < n;_k++) { - shiftRows(mat); - } - flip(mat,mat,0); - }else{ - for(int _k=0; _k < n;_k++) { - shiftRows(mat); - } - } - } - - //circular shift n columns from left to right if n > 0, -n columns from right to left if n < 0 - void TrackerKCFImpl::shiftCols(Mat& mat, int n) const { - if(n < 0){ - n = -n; - flip(mat,mat,1); - transpose(mat,mat); - shiftRows(mat,n); - transpose(mat,mat); - flip(mat,mat,1); - }else{ - transpose(mat,mat); - shiftRows(mat,n); - transpose(mat,mat); - } - } - - /* - * calculate the detection response - */ - void TrackerKCFImpl::calcResponse(const Mat alphaf_data, const Mat kf_data, Mat & response_data, Mat & spec_data) const { - //alpha f--> 2channels ; k --> 1 channel; - mulSpectrums(alphaf_data,kf_data,spec_data,0,false); - ifft2(spec_data,response_data); - } - - /* - * calculate the detection response for splitted form - */ - void TrackerKCFImpl::calcResponse(const Mat alphaf_data, const Mat _alphaf_den, const Mat kf_data, Mat & response_data, Mat & spec_data, Mat & spec2_data) const { - - mulSpectrums(alphaf_data,kf_data,spec_data,0,false); - - //z=(a+bi)/(c+di)=[(ac+bd)+i(bc-ad)]/(c^2+d^2) - float den; - for(int i=0;i(i,j)[0]*_alphaf_den.at(i,j)[0]+_alphaf_den.at(i,j)[1]*_alphaf_den.at(i,j)[1]); - spec2_data.at(i,j)[0]= - (spec_data.at(i,j)[0]*_alphaf_den.at(i,j)[0]+spec_data.at(i,j)[1]*_alphaf_den.at(i,j)[1])*den; - spec2_data.at(i,j)[1]= - (spec_data.at(i,j)[1]*_alphaf_den.at(i,j)[0]-spec_data.at(i,j)[0]*_alphaf_den.at(i,j)[1])*den; - } + bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE + { + impl.init(image, boundingBox); + model = impl.model; + sampler = makePtr(); + featureSet = makePtr(); + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE + { + Rect bb; + bool res = impl.update(image, bb); + boundingBox = bb; + return res; } - - ifft2(spec2_data,response_data); - } - - void TrackerKCFImpl::setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func){ - if(pca_func){ - extractor_pca.push_back(f); - use_custom_extractor_pca = true; - }else{ - extractor_npca.push_back(f); - use_custom_extractor_npca = true; + void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE + { + impl.setFeatureExtractor(f, pca_func); } - } - /*----------------------------------------------------------------------*/ +}; - /* - * Parameters - */ - TrackerKCF::Params::Params(){ - detect_thresh = 0.5f; - sigma=0.2f; - lambda=0.0001f; - interp_factor=0.075f; - output_sigma_factor=1.0f / 16.0f; - resize=true; - max_patch_size=80*80; - split_coeff=true; - wrap_kernel=false; - desc_npca = GRAY; - desc_pca = CN; +} // namespace - //feature compression - compress_feature=true; - compressed_size=2; - pca_learning_rate=0.15f; - } - - void TrackerKCF::Params::read( const cv::FileNode& fn ){ +void legacy::TrackerKCF::Params::read(const cv::FileNode& fn) +{ *this = TrackerKCF::Params(); if (!fn["detect_thresh"].empty()) @@ -970,9 +138,10 @@ namespace cv{ if (!fn["pca_learning_rate"].empty()) fn["pca_learning_rate"] >> pca_learning_rate; - } +} - void TrackerKCF::Params::write( cv::FileStorage& fs ) const{ +void legacy::TrackerKCF::Params::write(cv::FileStorage& fs) const +{ fs << "detect_thresh" << detect_thresh; fs << "sigma" << sigma; fs << "lambda" << lambda; @@ -987,5 +156,18 @@ namespace cv{ fs << "compress_feature" << compress_feature; fs << "compressed_size" << compressed_size; fs << "pca_learning_rate" << pca_learning_rate; - } -} /* namespace cv */ +} + + +}} // namespace legacy::tracking + +Ptr legacy::TrackerKCF::create(const legacy::TrackerKCF::Params ¶meters) +{ + return makePtr(parameters); +} +Ptr legacy::TrackerKCF::create() +{ + return create(legacy::TrackerKCF::Params()); +} + +} diff --git a/modules/tracking/src/legacy/trackerMIL.legacy.hpp b/modules/tracking/src/legacy/trackerMIL.legacy.hpp index 8ccc3b3b3..8e3a1b430 100644 --- a/modules/tracking/src/legacy/trackerMIL.legacy.hpp +++ b/modules/tracking/src/legacy/trackerMIL.legacy.hpp @@ -39,47 +39,54 @@ // //M*/ -#include "precomp.hpp" -#include "trackerMILModel.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" -namespace cv -{ +namespace cv { +namespace legacy { +inline namespace tracking { +namespace impl { -class TrackerMILImpl : public TrackerMIL +class TrackerMILImpl CV_FINAL : public legacy::TrackerMIL { - public: - TrackerMILImpl( const TrackerMIL::Params ¶meters = TrackerMIL::Params() ); - void read( const FileNode& fn ) CV_OVERRIDE; - void write( FileStorage& fs ) const CV_OVERRIDE; - - protected: - - bool initImpl( const Mat& image, const Rect2d& boundingBox ) CV_OVERRIDE; - bool updateImpl( const Mat& image, Rect2d& boundingBox ) CV_OVERRIDE; - void compute_integral( const Mat & img, Mat & ii_img ); - - TrackerMIL::Params params; +public: + cv::tracking::impl::TrackerMILImpl impl; + + TrackerMILImpl(const legacy::TrackerMIL::Params ¶meters) + : impl(parameters) + { + isInit = false; + } + + void read(const FileNode& fn) CV_OVERRIDE + { + static_cast(impl.params).read(fn); + } + void write(FileStorage& fs) const CV_OVERRIDE + { + static_cast(impl.params).write(fs); + } + + bool initImpl(const Mat& image, const Rect2d& boundingBox) CV_OVERRIDE + { + impl.init(image, boundingBox); + model = impl.model; + featureSet = impl.featureSet; + sampler = impl.sampler; + isInit = true; + return true; + } + bool updateImpl(const Mat& image, Rect2d& boundingBox) CV_OVERRIDE + { + Rect bb; + bool res = impl.update(image, bb); + boundingBox = bb; + return res; + } }; -/* - * TrackerMIL - */ - -/* - * Parameters - */ -TrackerMIL::Params::Params() -{ - samplerInitInRadius = 3; - samplerSearchWinSize = 25; - samplerInitMaxNegNum = 65; - samplerTrackInRadius = 4; - samplerTrackMaxPosNum = 100000; - samplerTrackMaxNegNum = 65; - featureSetNumFeatures = 250; -} +} // namespace -void TrackerMIL::Params::read( const cv::FileNode& fn ) +void legacy::TrackerMIL::Params::read(const cv::FileNode& fn) { samplerInitInRadius = fn["samplerInitInRadius"]; samplerSearchWinSize = fn["samplerSearchWinSize"]; @@ -90,7 +97,7 @@ void TrackerMIL::Params::read( const cv::FileNode& fn ) featureSetNumFeatures = fn["featureSetNumFeatures"]; } -void TrackerMIL::Params::write( cv::FileStorage& fs ) const +void legacy::TrackerMIL::Params::write(cv::FileStorage& fs) const { fs << "samplerInitInRadius" << samplerInitInRadius; fs << "samplerSearchWinSize" << samplerSearchWinSize; @@ -99,190 +106,17 @@ void TrackerMIL::Params::write( cv::FileStorage& fs ) const fs << "samplerTrackMaxPosNum" << samplerTrackMaxPosNum; fs << "samplerTrackMaxNegNum" << samplerTrackMaxNegNum; fs << "featureSetNumFeatures" << featureSetNumFeatures; - } -/* - * Constructor - */ -Ptr TrackerMIL::create(const TrackerMIL::Params ¶meters){ - return Ptr(new TrackerMILImpl(parameters)); -} -Ptr TrackerMIL::create(){ - return Ptr(new TrackerMILImpl()); -} -TrackerMILImpl::TrackerMILImpl( const TrackerMIL::Params ¶meters ) : - params( parameters ) -{ - isInit = false; -} +}} // namespace -void TrackerMILImpl::read( const cv::FileNode& fn ) +Ptr legacy::TrackerMIL::create(const legacy::TrackerMIL::Params ¶meters) { - params.read( fn ); -} - -void TrackerMILImpl::write( cv::FileStorage& fs ) const -{ - params.write( fs ); -} - -void TrackerMILImpl::compute_integral( const Mat & img, Mat & ii_img ) -{ - Mat ii; - std::vector ii_imgs; - integral( img, ii, CV_32F ); - split( ii, ii_imgs ); - ii_img = ii_imgs[0]; -} - -bool TrackerMILImpl::initImpl( const Mat& image, const Rect2d& boundingBox ) -{ - srand (1); - Mat intImage; - compute_integral( image, intImage ); - TrackerSamplerCSC::Params CSCparameters; - CSCparameters.initInRad = params.samplerInitInRadius; - CSCparameters.searchWinSize = params.samplerSearchWinSize; - CSCparameters.initMaxNegNum = params.samplerInitMaxNegNum; - CSCparameters.trackInPosRad = params.samplerTrackInRadius; - CSCparameters.trackMaxPosNum = params.samplerTrackMaxPosNum; - CSCparameters.trackMaxNegNum = params.samplerTrackMaxNegNum; - - Ptr CSCSampler = Ptr( new TrackerSamplerCSC( CSCparameters ) ); - if( !sampler->addTrackerSamplerAlgorithm( CSCSampler ) ) - return false; - - //or add CSC sampler with default parameters - //sampler->addTrackerSamplerAlgorithm( "CSC" ); - - //Positive sampling - CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); - sampler->sampling( intImage, boundingBox ); - std::vector posSamples = sampler->getSamples(); - - //Negative sampling - CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); - sampler->sampling( intImage, boundingBox ); - std::vector negSamples = sampler->getSamples(); - - if( posSamples.empty() || negSamples.empty() ) - return false; - - //compute HAAR features - TrackerFeatureHAAR::Params HAARparameters; - HAARparameters.numFeatures = params.featureSetNumFeatures; - HAARparameters.rectSize = Size( (int)boundingBox.width, (int)boundingBox.height ); - HAARparameters.isIntegral = true; - Ptr trackerFeature = Ptr( new TrackerFeatureHAAR( HAARparameters ) ); - featureSet->addTrackerFeature( trackerFeature ); - - featureSet->extraction( posSamples ); - const std::vector posResponse = featureSet->getResponses(); - - featureSet->extraction( negSamples ); - const std::vector negResponse = featureSet->getResponses(); - - model = Ptr( new TrackerMILModel( boundingBox ) ); - Ptr stateEstimator = Ptr( - new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ) ); - model->setTrackerStateEstimator( stateEstimator ); - - //Run model estimation and update - model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); - model->modelEstimation( posResponse ); - model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); - model->modelEstimation( negResponse ); - model->modelUpdate(); - - return true; + return makePtr(parameters); } - -bool TrackerMILImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) +Ptr legacy::TrackerMIL::create() { - Mat intImage; - compute_integral( image, intImage ); - - //get the last location [AAM] X(k-1) - Ptr lastLocation = model->getLastTargetState(); - Rect lastBoundingBox( (int)lastLocation->getTargetPosition().x, (int)lastLocation->getTargetPosition().y, lastLocation->getTargetWidth(), - lastLocation->getTargetHeight() ); - - //sampling new frame based on last location - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_DETECT ); - sampler->sampling( intImage, lastBoundingBox ); - std::vector detectSamples = sampler->getSamples(); - if( detectSamples.empty() ) - return false; - - /*//TODO debug samples - Mat f; - image.copyTo(f); - - for( size_t i = 0; i < detectSamples.size(); i=i+10 ) - { - Size sz; - Point off; - detectSamples.at(i).locateROI(sz, off); - rectangle(f, Rect(off.x,off.y,detectSamples.at(i).cols,detectSamples.at(i).rows), Scalar(255,0,0), 1); - }*/ - - //extract features from new samples - featureSet->extraction( detectSamples ); - std::vector response = featureSet->getResponses(); - - //predict new location - ConfidenceMap cmap; - model.staticCast()->setMode( TrackerMILModel::MODE_ESTIMATON, detectSamples ); - model.staticCast()->responseToConfidenceMap( response, cmap ); - model->getTrackerStateEstimator().staticCast()->setCurrentConfidenceMap( cmap ); - - if( !model->runStateEstimator() ) - { - return false; - } - - Ptr currentState = model->getLastTargetState(); - boundingBox = Rect( (int)currentState->getTargetPosition().x, (int)currentState->getTargetPosition().y, currentState->getTargetWidth(), - currentState->getTargetHeight() ); - - /*//TODO debug - rectangle(f, lastBoundingBox, Scalar(0,255,0), 1); - rectangle(f, boundingBox, Scalar(0,0,255), 1); - imshow("f", f); - //waitKey( 0 );*/ - - //sampling new frame based on new location - //Positive sampling - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); - sampler->sampling( intImage, boundingBox ); - std::vector posSamples = sampler->getSamples(); - - //Negative sampling - ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); - sampler->sampling( intImage, boundingBox ); - std::vector negSamples = sampler->getSamples(); - - if( posSamples.empty() || negSamples.empty() ) - return false; - - //extract features - featureSet->extraction( posSamples ); - std::vector posResponse = featureSet->getResponses(); - - featureSet->extraction( negSamples ); - std::vector negResponse = featureSet->getResponses(); - - //model estimate - model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); - model->modelEstimation( posResponse ); - model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); - model->modelEstimation( negResponse ); - - //model update - model->modelUpdate(); - - return true; + return create(legacy::TrackerMIL::Params()); } -} /* namespace cv */ +} // namespace diff --git a/modules/tracking/src/mosseTracker.cpp b/modules/tracking/src/mosseTracker.cpp index 038a8ba1d..b972bf93d 100644 --- a/modules/tracking/src/mosseTracker.cpp +++ b/modules/tracking/src/mosseTracker.cpp @@ -13,24 +13,28 @@ // Cracki: for the idea of only converting the used patch to gray // -#include "opencv2/tracking.hpp" +#include "precomp.hpp" + +#include "opencv2/tracking/tracking_legacy.hpp" namespace cv { -namespace tracking { +inline namespace tracking { +namespace impl { +namespace { -struct DummyModel : TrackerModel +struct DummyModel : detail::tracking::TrackerModel { virtual void modelUpdateImpl() CV_OVERRIDE {} virtual void modelEstimationImpl( const std::vector& ) CV_OVERRIDE {} }; - const double eps=0.00001; // for normalization const double rate=0.2; // learning rate const double psrThreshold=5.7; // no detection, if PSR is smaller than this +} // namespace -struct MosseImpl CV_FINAL : TrackerMOSSE +struct MosseImpl CV_FINAL : legacy::TrackerMOSSE { protected: @@ -237,13 +241,12 @@ public: }; // MosseImpl -} // tracking - +}} // namespace -Ptr TrackerMOSSE::create() +Ptr legacy::tracking::TrackerMOSSE::create() { - return makePtr(); + return makePtr(); } -} // cv +} // namespace diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 7b71bccde..963e6eb65 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -39,10 +39,17 @@ // //M*/ +#include "precomp.hpp" #include "multiTracker.hpp" -namespace cv -{ +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { + +using namespace impl; + //Multitracker bool MultiTracker_Alt::addTarget(InputArray image, const Rect2d& boundingBox, Ptr tracker_algorithm) { @@ -249,12 +256,18 @@ namespace cv return success; } +}} // namespace + + +inline namespace tracking { +namespace impl { + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, - std::vector > &trackers) + std::vector > &trackers) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[0]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + legacy::Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); @@ -445,11 +458,11 @@ namespace cv #ifdef HAVE_OPENCL void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, - std::vector > &trackers) + std::vector > &trackers) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[0]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + legacy::Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); @@ -656,4 +669,4 @@ namespace cv } #endif -} +}}} // namespace diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index 4ab654e9b..504fafd22 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -42,18 +42,18 @@ #ifndef OPENCV_MULTITRACKER #define OPENCV_MULTITRACKER -#include "precomp.hpp" #include "tldTracker.hpp" #include "tldUtils.hpp" #include -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, - std::vector& detect_flgs, std::vector >& trackers); + std::vector& detect_flgs, std::vector >& trackers); #ifdef HAVE_OPENCL void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, - std::vector& detect_flgs, std::vector >& trackers); + std::vector& detect_flgs, std::vector >& trackers); #endif -} +}}} // namespace #endif \ No newline at end of file diff --git a/modules/tracking/src/multiTracker_alt.cpp b/modules/tracking/src/multiTracker_alt.cpp index 749747977..d5a96e7cc 100644 --- a/modules/tracking/src/multiTracker_alt.cpp +++ b/modules/tracking/src/multiTracker_alt.cpp @@ -40,8 +40,11 @@ //M*/ #include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" namespace cv { +namespace legacy { +inline namespace tracking { // constructor MultiTracker::MultiTracker(){}; @@ -105,4 +108,4 @@ namespace cv { return makePtr(); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/onlineBoosting.cpp b/modules/tracking/src/onlineBoosting.cpp index 3254c0b5c..800d7ee99 100644 --- a/modules/tracking/src/onlineBoosting.cpp +++ b/modules/tracking/src/onlineBoosting.cpp @@ -42,8 +42,9 @@ #include "precomp.hpp" #include "opencv2/tracking/onlineBoosting.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { StrongClassifierDirectSelection::StrongClassifierDirectSelection( int numBaseClf, int numWeakClf, Size patchSz, const Rect& sampleROI, bool useFeatureEx, int iterationInit ) @@ -732,4 +733,4 @@ int ClassifierThreshold::eval( float value ) return ( ( ( m_parity * ( value - m_threshold ) ) > 0 ) ? 1 : -1 ); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/precomp.hpp b/modules/tracking/src/precomp.hpp index f892f8678..50ebd7272 100644 --- a/modules/tracking/src/precomp.hpp +++ b/modules/tracking/src/precomp.hpp @@ -42,18 +42,29 @@ #ifndef __OPENCV_PRECOMP_H__ #define __OPENCV_PRECOMP_H__ -#include "cvconfig.h" -#include "opencv2/tracking.hpp" -#include "opencv2/core/utility.hpp" +#include "opencv2/core.hpp" #include "opencv2/core/ocl.hpp" -#include #include "opencv2/core/hal/hal.hpp" -namespace cv -{ - extern const float ColorNames[][10]; +#include "opencv2/video/tracking.hpp" + +#include "opencv2/tracking.hpp" + - namespace tracking { +#include "opencv2/tracking/tracking_internals.hpp" + +namespace cv { inline namespace tracking { +namespace impl { } +using namespace impl; +using namespace cv::detail::tracking; +}} // namespace + + +namespace cv { +namespace detail { +inline namespace tracking { + + extern const float ColorNames[][10]; /* Cholesky decomposition The function performs Cholesky decomposition . @@ -102,7 +113,6 @@ namespace cv return success; } - } // tracking -} // cv +}}} // namespace #endif diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 97b29645f..6ce0a9785 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -39,10 +39,13 @@ // //M*/ +#include "precomp.hpp" #include "opencv2/tracking/tldDataset.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { + namespace tld { char tldRootPath[100]; @@ -182,4 +185,5 @@ namespace cv } } -} + +}}} diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index bb52a4899..826882135 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -39,15 +39,15 @@ // //M*/ +#include "precomp.hpp" + #include "tldDetector.hpp" #include "tracking_utils.hpp" -#include - -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { // Calculate offsets for classifiers void TLDDetector::prepareClassifiers(int rowstep) { @@ -619,5 +619,4 @@ namespace cv return ((p2 - p * p) > VARIANCE_THRESHOLD * *originalVariance); } - } -} +}}}} // namespace diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 292cbbb35..a2b329d6e 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -42,15 +42,15 @@ #ifndef OPENCV_TLD_DETECTOR #define OPENCV_TLD_DETECTOR -#include "precomp.hpp" #include "opencl_kernels_tracking.hpp" #include "tldEnsembleClassifier.hpp" #include "tldUtils.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + const int STANDARD_PATCH_SIZE = 15; const int NEG_EXAMPLES_IN_INIT_MODEL = 300; const int MAX_EXAMPLES_IN_MODEL = 500; @@ -116,7 +116,6 @@ namespace cv }; - } -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldEnsembleClassifier.cpp b/modules/tracking/src/tldEnsembleClassifier.cpp index 1d72b3e81..c800fc3c9 100644 --- a/modules/tracking/src/tldEnsembleClassifier.cpp +++ b/modules/tracking/src/tldEnsembleClassifier.cpp @@ -39,12 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldEnsembleClassifier.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + // Constructor TLDEnsembleClassifier::TLDEnsembleClassifier(const std::vector& meas, int beg, int end) :lastStep_(-1) { @@ -194,5 +196,4 @@ namespace cv return (int)classifiers.size(); } - } -} \ No newline at end of file +}}}} // namespace diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp index f0ec175ba..6df65e481 100644 --- a/modules/tracking/src/tldEnsembleClassifier.hpp +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -40,12 +40,12 @@ //M*/ #include -#include "precomp.hpp" -namespace cv -{ - namespace tld - { +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + class TLDEnsembleClassifier { public: @@ -64,5 +64,5 @@ namespace cv std::vector offset; int lastStep_; }; - } -} \ No newline at end of file + +}}}} // namespace diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 833e586d0..ad787ab8e 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -39,14 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldModel.hpp" -#include +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace cv -{ - namespace tld - { //Constructor TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize): timeStampPositiveNext(0), timeStampNegativeNext(0), minSize_(minSize), params_(params), boundingBox_(boundingBox) @@ -361,5 +361,5 @@ namespace cv dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); } - } -} + +}}}} // namespace diff --git a/modules/tracking/src/tldModel.hpp b/modules/tracking/src/tldModel.hpp index 7616ebdc4..57331756c 100644 --- a/modules/tracking/src/tldModel.hpp +++ b/modules/tracking/src/tldModel.hpp @@ -45,10 +45,15 @@ #include "tldDetector.hpp" #include "tldUtils.hpp" -namespace cv -{ - namespace tld - { +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { + +using namespace cv::legacy; + class TrackerTLDModel : public TrackerModel { public: @@ -84,7 +89,6 @@ namespace cv RNG rng; }; - } -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index db1fccf54..92038ef2d 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -39,10 +39,15 @@ // //M*/ +#include "precomp.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" #include "tldTracker.hpp" -namespace cv -{ +namespace cv { +namespace legacy { +inline namespace tracking { +using namespace impl; +using namespace impl::tld; TrackerTLD::Params::Params(){} @@ -60,8 +65,11 @@ Ptr TrackerTLD::create() return Ptr(new tld::TrackerTLDImpl()); } -namespace tld -{ +}} // namespace + +inline namespace tracking { +namespace impl { +namespace tld { TrackerTLDImpl::TrackerTLDImpl(const TrackerTLD::Params ¶meters) : params( parameters ) @@ -323,6 +331,4 @@ void Data::printme(FILE* port) dfprintf((port, "\tminSize = %dx%d\n", minSize.width, minSize.height)); } -} - -} +}}}} // namespace diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 3f9861192..e96b3e04d 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -42,18 +42,17 @@ #ifndef OPENCV_TLD_TRACKER #define OPENCV_TLD_TRACKER -#include "precomp.hpp" #include "opencv2/video/tracking.hpp" #include "opencv2/imgproc.hpp" #include "tldModel.hpp" -#include -#include +#include +#include -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace tld -{ class TrackerProxy { public: @@ -168,7 +167,6 @@ public: }; -} -} +}}}} // namespace #endif diff --git a/modules/tracking/src/tldUtils.cpp b/modules/tracking/src/tldUtils.cpp index b570a1f3b..5dca7b5f8 100644 --- a/modules/tracking/src/tldUtils.cpp +++ b/modules/tracking/src/tldUtils.cpp @@ -39,13 +39,14 @@ // //M*/ +#include "precomp.hpp" #include "tldUtils.hpp" -namespace cv -{ -namespace tld -{ +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { //Debug functions and variables Rect2d etalon(14.0, 110.0, 20.0, 20.0); @@ -192,4 +193,4 @@ void resample(const Mat& img, const Rect2d& r2, Mat_& samples) } -}} +}}}} // namespace diff --git a/modules/tracking/src/tldUtils.hpp b/modules/tracking/src/tldUtils.hpp index 1a606429d..311e08f5e 100644 --- a/modules/tracking/src/tldUtils.hpp +++ b/modules/tracking/src/tldUtils.hpp @@ -1,12 +1,11 @@ #ifndef OPENCV_TLD_UTILS #define OPENCV_TLD_UTILS -#include "precomp.hpp" +namespace cv { +inline namespace tracking { +namespace impl { +namespace tld { -namespace cv -{ - namespace tld - { //debug functions and variables #define ALEX_DEBUG #ifdef ALEX_DEBUG @@ -48,7 +47,7 @@ namespace cv double variance(const Mat& img); void getClosestN(std::vector& scanGrid, Rect2d bBox, int n, std::vector& res); double scaleAndBlur(const Mat& originalImg, int scale, Mat& scaledImg, Mat& blurredImg, Size GaussBlurKernelSize, double scaleStep); - } -} + +}}}} // namespace #endif diff --git a/modules/tracking/src/tracker.cpp b/modules/tracking/src/tracker.cpp new file mode 100644 index 000000000..d6c0a8d0a --- /dev/null +++ b/modules/tracking/src/tracker.cpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +namespace cv { +inline namespace tracking { + +Tracker::Tracker() +{ + // nothing +} + +Tracker::~Tracker() +{ + // nothing +} + +}} // namespace + + +#include "legacy/tracker.legacy.hpp" diff --git a/modules/tracking/src/trackerBoosting.cpp b/modules/tracking/src/trackerBoosting.cpp index 05b11aa2d..e3dcc1998 100644 --- a/modules/tracking/src/trackerBoosting.cpp +++ b/modules/tracking/src/trackerBoosting.cpp @@ -42,8 +42,12 @@ #include "precomp.hpp" #include "trackerBoostingModel.hpp" -namespace cv -{ +#include "opencv2/tracking/tracking_legacy.hpp" + +namespace cv { +namespace legacy { +inline namespace tracking { +using namespace impl; class TrackerBoostingImpl : public TrackerBoosting { @@ -319,4 +323,4 @@ bool TrackerBoostingImpl::updateImpl( const Mat& image, Rect2d& boundingBox ) return true; } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerBoostingModel.cpp b/modules/tracking/src/trackerBoostingModel.cpp index bd6148a9d..3fb48f095 100644 --- a/modules/tracking/src/trackerBoostingModel.cpp +++ b/modules/tracking/src/trackerBoostingModel.cpp @@ -39,14 +39,16 @@ // //M*/ +#include "precomp.hpp" #include "trackerBoostingModel.hpp" /** * TrackerBoostingModel */ -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { TrackerBoostingModel::TrackerBoostingModel( const Rect& boundingBox ) { @@ -119,4 +121,4 @@ void TrackerBoostingModel::responseToConfidenceMap( const std::vector& resp } } -} +}}} // namespace diff --git a/modules/tracking/src/trackerBoostingModel.hpp b/modules/tracking/src/trackerBoostingModel.hpp index 0d4582cd1..45aa1dc6d 100644 --- a/modules/tracking/src/trackerBoostingModel.hpp +++ b/modules/tracking/src/trackerBoostingModel.hpp @@ -42,11 +42,9 @@ #ifndef __OPENCV_TRACKER_BOOSTING_MODEL_HPP__ #define __OPENCV_TRACKER_BOOSTING_MODEL_HPP__ -#include "precomp.hpp" -#include "opencv2/core.hpp" - -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { /** * \brief Implementation of TrackerModel for BOOSTING algorithm @@ -103,6 +101,6 @@ class TrackerBoostingModel : public TrackerModel int mode; }; -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/src/trackerCSRT.cpp b/modules/tracking/src/trackerCSRT.cpp new file mode 100644 index 000000000..1b3f48aa7 --- /dev/null +++ b/modules/tracking/src/trackerCSRT.cpp @@ -0,0 +1,655 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "trackerCSRTSegmentation.hpp" +#include "trackerCSRTUtils.hpp" +#include "trackerCSRTScaleEstimation.hpp" + +namespace cv { +inline namespace tracking { +namespace impl { + +/** +* \brief Implementation of TrackerModel for CSRT algorithm +*/ +class TrackerCSRTModel CV_FINAL : public TrackerModel +{ +public: + TrackerCSRTModel(){} + ~TrackerCSRTModel(){} +protected: + void modelEstimationImpl(const std::vector& /*responses*/) CV_OVERRIDE {} + void modelUpdateImpl() CV_OVERRIDE {} +}; + +class TrackerCSRTImpl CV_FINAL : public TrackerCSRT +{ +public: + TrackerCSRTImpl(const Params ¶meters = Params()); + + Params params; + + Ptr model; + + // Tracker API + virtual void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + virtual bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + virtual void setInitialMask(InputArray mask) CV_OVERRIDE; + +protected: + void update_csr_filter(const Mat &image, const Mat &my_mask); + void update_histograms(const Mat &image, const Rect ®ion); + void extract_histograms(const Mat &image, cv::Rect region, Histogram &hf, Histogram &hb); + std::vector create_csr_filter(const std::vector + img_features, const cv::Mat Y, const cv::Mat P); + Mat calculate_response(const Mat &image, const std::vector filter); + Mat get_location_prior(const Rect roi, const Size2f target_size, const Size img_sz); + Mat segment_region(const Mat &image, const Point2f &object_center, + const Size2f &template_size, const Size &target_size, float scale_factor); + Point2f estimate_new_position(const Mat &image); + std::vector get_features(const Mat &patch, const Size2i &feature_size); + + bool check_mask_area(const Mat &mat, const double obj_area); + float current_scale_factor; + Mat window; + Mat yf; + Rect2f bounding_box; + std::vector csr_filter; + std::vector filter_weights; + Size2f original_target_size; + Size2i image_size; + Size2f template_size; + Size2i rescaled_template_size; + float rescale_ratio; + Point2f object_center; + DSST dsst; + Histogram hist_foreground; + Histogram hist_background; + double p_b; + Mat erode_element; + Mat filter_mask; + Mat preset_mask; + Mat default_mask; + float default_mask_area; + int cell_size; +}; + +TrackerCSRTImpl::TrackerCSRTImpl(const TrackerCSRT::Params ¶meters) : + params(parameters) +{ + // nothing +} + +void TrackerCSRTImpl::setInitialMask(InputArray mask) +{ + preset_mask = mask.getMat(); +} + +bool TrackerCSRTImpl::check_mask_area(const Mat &mat, const double obj_area) +{ + double threshold = 0.05; + double mask_area= sum(mat)[0]; + if(mask_area < threshold*obj_area) { + return false; + } + return true; +} + +Mat TrackerCSRTImpl::calculate_response(const Mat &image, const std::vector filter) +{ + Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), + cvFloor(current_scale_factor * template_size.height)); + resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); + + std::vector ftrs = get_features(patch, yf.size()); + std::vector Ffeatures = fourier_transform_features(ftrs); + Mat resp, res; + if(params.use_channel_weights){ + res = Mat::zeros(Ffeatures[0].size(), CV_32FC2); + Mat resp_ch; + Mat mul_mat; + for(size_t i = 0; i < Ffeatures.size(); ++i) { + mulSpectrums(Ffeatures[i], filter[i], resp_ch, 0, true); + res += (resp_ch * filter_weights[i]); + } + idft(res, res, DFT_SCALE | DFT_REAL_OUTPUT); + } else { + res = Mat::zeros(Ffeatures[0].size(), CV_32FC2); + Mat resp_ch; + for(size_t i = 0; i < Ffeatures.size(); ++i) { + mulSpectrums(Ffeatures[i], filter[i], resp_ch, 0 , true); + res = res + resp_ch; + } + idft(res, res, DFT_SCALE | DFT_REAL_OUTPUT); + } + return res; +} + +void TrackerCSRTImpl::update_csr_filter(const Mat &image, const Mat &mask) +{ + Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), + cvFloor(current_scale_factor * template_size.height)); + resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); + + std::vector ftrs = get_features(patch, yf.size()); + std::vector Fftrs = fourier_transform_features(ftrs); + std::vector new_csr_filter = create_csr_filter(Fftrs, yf, mask); + //calculate per channel weights + if(params.use_channel_weights) { + Mat current_resp; + double max_val; + float sum_weights = 0; + std::vector new_filter_weights = std::vector(new_csr_filter.size()); + for(size_t i = 0; i < new_csr_filter.size(); ++i) { + mulSpectrums(Fftrs[i], new_csr_filter[i], current_resp, 0, true); + idft(current_resp, current_resp, DFT_SCALE | DFT_REAL_OUTPUT); + minMaxLoc(current_resp, NULL, &max_val, NULL, NULL); + sum_weights += static_cast(max_val); + new_filter_weights[i] = static_cast(max_val); + } + //update filter weights with new values + float updated_sum = 0; + for(size_t i = 0; i < filter_weights.size(); ++i) { + filter_weights[i] = filter_weights[i]*(1.0f - params.weights_lr) + + params.weights_lr * (new_filter_weights[i] / sum_weights); + updated_sum += filter_weights[i]; + } + //normalize weights + for(size_t i = 0; i < filter_weights.size(); ++i) { + filter_weights[i] /= updated_sum; + } + } + for(size_t i = 0; i < csr_filter.size(); ++i) { + csr_filter[i] = (1.0f - params.filter_lr)*csr_filter[i] + params.filter_lr * new_csr_filter[i]; + } + std::vector().swap(ftrs); + std::vector().swap(Fftrs); +} + + +std::vector TrackerCSRTImpl::get_features(const Mat &patch, const Size2i &feature_size) +{ + std::vector features; + if (params.use_hog) { + std::vector hog = get_features_hog(patch, cell_size); + features.insert(features.end(), hog.begin(), + hog.begin()+params.num_hog_channels_used); + } + if (params.use_color_names) { + std::vector cn; + cn = get_features_cn(patch, feature_size); + features.insert(features.end(), cn.begin(), cn.end()); + } + if(params.use_gray) { + Mat gray_m; + cvtColor(patch, gray_m, COLOR_BGR2GRAY); + resize(gray_m, gray_m, feature_size, 0, 0, INTER_CUBIC); + gray_m.convertTo(gray_m, CV_32FC1, 1.0/255.0, -0.5); + features.push_back(gray_m); + } + if(params.use_rgb) { + std::vector rgb_features = get_features_rgb(patch, feature_size); + features.insert(features.end(), rgb_features.begin(), rgb_features.end()); + } + + for (size_t i = 0; i < features.size(); ++i) { + features.at(i) = features.at(i).mul(window); + } + return features; +} + +class ParallelCreateCSRFilter : public ParallelLoopBody { +public: + ParallelCreateCSRFilter( + const std::vector img_features, + const cv::Mat Y, + const cv::Mat P, + int admm_iterations, + std::vector &result_filter_): + result_filter(result_filter_) + { + this->img_features = img_features; + this->Y = Y; + this->P = P; + this->admm_iterations = admm_iterations; + } + virtual void operator ()(const Range& range) const CV_OVERRIDE + { + for (int i = range.start; i < range.end; i++) { + float mu = 5.0f; + float beta = 3.0f; + float mu_max = 20.0f; + float lambda = mu / 100.0f; + + Mat F = img_features[i]; + + Mat Sxy, Sxx; + mulSpectrums(F, Y, Sxy, 0, true); + mulSpectrums(F, F, Sxx, 0, true); + + Mat H; + H = divide_complex_matrices(Sxy, (Sxx + lambda)); + idft(H, H, DFT_SCALE|DFT_REAL_OUTPUT); + H = H.mul(P); + dft(H, H, DFT_COMPLEX_OUTPUT); + Mat L = Mat::zeros(H.size(), H.type()); //Lagrangian multiplier + Mat G; + for(int iteration = 0; iteration < admm_iterations; ++iteration) { + G = divide_complex_matrices((Sxy + (mu * H) - L) , (Sxx + mu)); + idft((mu * G) + L, H, DFT_SCALE | DFT_REAL_OUTPUT); + float lm = 1.0f / (lambda+mu); + H = H.mul(P*lm); + dft(H, H, DFT_COMPLEX_OUTPUT); + + //Update variables for next iteration + L = L + mu * (G - H); + mu = min(mu_max, beta*mu); + } + result_filter[i] = H; + } + } + + ParallelCreateCSRFilter& operator=(const ParallelCreateCSRFilter &) { + return *this; + } + +private: + int admm_iterations; + Mat Y; + Mat P; + std::vector img_features; + std::vector &result_filter; +}; + + +std::vector TrackerCSRTImpl::create_csr_filter( + const std::vector img_features, + const cv::Mat Y, + const cv::Mat P) +{ + std::vector result_filter; + result_filter.resize(img_features.size()); + ParallelCreateCSRFilter parallelCreateCSRFilter(img_features, Y, P, + params.admm_iterations, result_filter); + parallel_for_(Range(0, static_cast(result_filter.size())), parallelCreateCSRFilter); + + return result_filter; +} + +Mat TrackerCSRTImpl::get_location_prior( + const Rect roi, + const Size2f target_size, + const Size img_sz) +{ + int x1 = cvRound(max(min(roi.x-1, img_sz.width-1) , 0)); + int y1 = cvRound(max(min(roi.y-1, img_sz.height-1) , 0)); + + int x2 = cvRound(min(max(roi.width-1, 0) , img_sz.width-1)); + int y2 = cvRound(min(max(roi.height-1, 0) , img_sz.height-1)); + + Size target_sz; + target_sz.width = target_sz.height = cvFloor(min(target_size.width, target_size.height)); + + double cx = x1 + (x2-x1)/2.; + double cy = y1 + (y2-y1)/2.; + double kernel_size_width = 1.0/(0.5*static_cast(target_sz.width)*1.4142+1); + double kernel_size_height = 1.0/(0.5*static_cast(target_sz.height)*1.4142+1); + + cv::Mat kernel_weight = Mat::zeros(1 + cvFloor(y2 - y1) , 1+cvFloor(-(x1-cx) + (x2-cx)), CV_64FC1); + for (int y = y1; y < y2+1; ++y){ + double * weightPtr = kernel_weight.ptr(y); + double tmp_y = std::pow((cy-y)*kernel_size_height, 2); + for (int x = x1; x < x2+1; ++x){ + weightPtr[x] = kernel_epan(std::pow((cx-x)*kernel_size_width,2) + tmp_y); + } + } + + double max_val; + cv::minMaxLoc(kernel_weight, NULL, &max_val, NULL, NULL); + Mat fg_prior = kernel_weight / max_val; + fg_prior.setTo(0.5, fg_prior < 0.5); + fg_prior.setTo(0.9, fg_prior > 0.9); + return fg_prior; +} + +Mat TrackerCSRTImpl::segment_region( + const Mat &image, + const Point2f &object_center, + const Size2f &template_size, + const Size &target_size, + float scale_factor) +{ + Rect valid_pixels; + Mat patch = get_subwindow(image, object_center, cvFloor(scale_factor * template_size.width), + cvFloor(scale_factor * template_size.height), &valid_pixels); + Size2f scaled_target = Size2f(target_size.width * scale_factor, + target_size.height * scale_factor); + Mat fg_prior = get_location_prior( + Rect(0,0, patch.size().width, patch.size().height), + scaled_target , patch.size()); + + std::vector img_channels; + split(patch, img_channels); + std::pair probs = Segment::computePosteriors2(img_channels, 0, 0, patch.cols, patch.rows, + p_b, fg_prior, 1.0-fg_prior, hist_foreground, hist_background); + + Mat mask = Mat::zeros(probs.first.size(), probs.first.type()); + probs.first(valid_pixels).copyTo(mask(valid_pixels)); + double max_resp = get_max(mask); + threshold(mask, mask, max_resp / 2.0, 1, THRESH_BINARY); + mask.convertTo(mask, CV_32FC1, 1.0); + return mask; +} + + +void TrackerCSRTImpl::extract_histograms(const Mat &image, cv::Rect region, Histogram &hf, Histogram &hb) +{ + // get coordinates of the region + int x1 = std::min(std::max(0, region.x), image.cols-1); + int y1 = std::min(std::max(0, region.y), image.rows-1); + int x2 = std::min(std::max(0, region.x + region.width), image.cols-1); + int y2 = std::min(std::max(0, region.y + region.height), image.rows-1); + + // calculate coordinates of the background region + int offsetX = (x2-x1+1) / params.background_ratio; + int offsetY = (y2-y1+1) / params.background_ratio; + int outer_y1 = std::max(0, (int)(y1-offsetY)); + int outer_y2 = std::min(image.rows, (int)(y2+offsetY+1)); + int outer_x1 = std::max(0, (int)(x1-offsetX)); + int outer_x2 = std::min(image.cols, (int)(x2+offsetX+1)); + + // calculate probability for the background + p_b = 1.0 - ((x2-x1+1) * (y2-y1+1)) / + ((double) (outer_x2-outer_x1+1) * (outer_y2-outer_y1+1)); + + // split multi-channel image into the std::vector of matrices + std::vector img_channels(image.channels()); + split(image, img_channels); + for(size_t k=0; k().swap(img_channels); +} + +void TrackerCSRTImpl::update_histograms(const Mat &image, const Rect ®ion) +{ + // create temporary histograms + Histogram hf(image.channels(), params.histogram_bins); + Histogram hb(image.channels(), params.histogram_bins); + extract_histograms(image, region, hf, hb); + + // get histogram vectors from temporary histograms + std::vector hf_vect_new = hf.getHistogramVector(); + std::vector hb_vect_new = hb.getHistogramVector(); + // get histogram vectors from learned histograms + std::vector hf_vect = hist_foreground.getHistogramVector(); + std::vector hb_vect = hist_background.getHistogramVector(); + + // update histograms - use learning rate + for(size_t i=0; i().swap(hf_vect); + std::vector().swap(hb_vect); +} + +Point2f TrackerCSRTImpl::estimate_new_position(const Mat &image) +{ + + Mat resp = calculate_response(image, csr_filter); + + double max_val; + Point max_loc; + minMaxLoc(resp, NULL, &max_val, NULL, &max_loc); + if (max_val < params.psr_threshold) + return Point2f(-1,-1); // target "lost" + + // take into account also subpixel accuracy + float col = ((float) max_loc.x) + subpixel_peak(resp, "horizontal", max_loc); + float row = ((float) max_loc.y) + subpixel_peak(resp, "vertical", max_loc); + if(row + 1 > (float)resp.rows / 2.0f) { + row = row - resp.rows; + } + if(col + 1 > (float)resp.cols / 2.0f) { + col = col - resp.cols; + } + // calculate x and y displacements + Point2f new_center = object_center + Point2f(current_scale_factor * (1.0f / rescale_ratio) *cell_size*(col), + current_scale_factor * (1.0f / rescale_ratio) *cell_size*(row)); + //sanity checks + if(new_center.x < 0) + new_center.x = 0; + if(new_center.x >= image_size.width) + new_center.x = static_cast(image_size.width - 1); + if(new_center.y < 0) + new_center.y = 0; + if(new_center.y >= image_size.height) + new_center.y = static_cast(image_size.height - 1); + + return new_center; +} + +// ********************************************************************* +// * Update API function * +// ********************************************************************* +bool TrackerCSRTImpl::update(InputArray image_, Rect& boundingBox) +{ + Mat image; + if(image_.channels() == 1) //treat gray image as color image + cvtColor(image_, image, COLOR_GRAY2BGR); + else + image = image_.getMat(); + + object_center = estimate_new_position(image); + if (object_center.x < 0 && object_center.y < 0) + return false; + + current_scale_factor = dsst.getScale(image, object_center); + //update bouding_box according to new scale and location + bounding_box.x = object_center.x - current_scale_factor * original_target_size.width / 2.0f; + bounding_box.y = object_center.y - current_scale_factor * original_target_size.height / 2.0f; + bounding_box.width = current_scale_factor * original_target_size.width; + bounding_box.height = current_scale_factor * original_target_size.height; + + //update tracker + if(params.use_segmentation) { + Mat hsv_img = bgr2hsv(image); + update_histograms(hsv_img, bounding_box); + filter_mask = segment_region(hsv_img, object_center, + template_size,original_target_size, current_scale_factor); + resize(filter_mask, filter_mask, yf.size(), 0, 0, INTER_NEAREST); + if(check_mask_area(filter_mask, default_mask_area)) { + dilate(filter_mask , filter_mask, erode_element); + } else { + filter_mask = default_mask; + } + } else { + filter_mask = default_mask; + } + update_csr_filter(image, filter_mask); + dsst.update(image, object_center); + boundingBox = bounding_box; + return true; +} + +// ********************************************************************* +// * Init API function * +// ********************************************************************* +void TrackerCSRTImpl::init(InputArray image_, const Rect& boundingBox) +{ + Mat image; + if(image_.channels() == 1) //treat gray image as color image + cvtColor(image_, image, COLOR_GRAY2BGR); + else + image = image_.getMat(); + + current_scale_factor = 1.0; + image_size = image.size(); + bounding_box = boundingBox; + cell_size = cvFloor(std::min(4.0, std::max(1.0, static_cast( + cvCeil((bounding_box.width * bounding_box.height)/400.0))))); + original_target_size = Size(bounding_box.size()); + + template_size.width = static_cast(cvFloor(original_target_size.width + params.padding * + sqrt(original_target_size.width * original_target_size.height))); + template_size.height = static_cast(cvFloor(original_target_size.height + params.padding * + sqrt(original_target_size.width * original_target_size.height))); + template_size.width = template_size.height = + (template_size.width + template_size.height) / 2.0f; + rescale_ratio = sqrt(pow(params.template_size,2) / (template_size.width * template_size.height)); + if(rescale_ratio > 1) { + rescale_ratio = 1; + } + rescaled_template_size = Size2i(cvFloor(template_size.width * rescale_ratio), + cvFloor(template_size.height * rescale_ratio)); + object_center = Point2f(static_cast(boundingBox.x) + original_target_size.width / 2.0f, + static_cast(boundingBox.y) + original_target_size.height / 2.0f); + + yf = gaussian_shaped_labels(params.gsl_sigma, + rescaled_template_size.width / cell_size, rescaled_template_size.height / cell_size); + if(params.window_function.compare("hann") == 0) { + window = get_hann_win(Size(yf.cols,yf.rows)); + } else if(params.window_function.compare("cheb") == 0) { + window = get_chebyshev_win(Size(yf.cols,yf.rows), params.cheb_attenuation); + } else if(params.window_function.compare("kaiser") == 0) { + window = get_kaiser_win(Size(yf.cols,yf.rows), params.kaiser_alpha); + } else { + CV_Error(Error::StsBadArg, "Not a valid window function"); + } + + Size2i scaled_obj_size = Size2i(cvFloor(original_target_size.width * rescale_ratio / cell_size), + cvFloor(original_target_size.height * rescale_ratio / cell_size)); + //set dummy mask and area; + int x0 = std::max((yf.size().width - scaled_obj_size.width)/2 - 1, 0); + int y0 = std::max((yf.size().height - scaled_obj_size.height)/2 - 1, 0); + default_mask = Mat::zeros(yf.size(), CV_32FC1); + default_mask(Rect(x0,y0,scaled_obj_size.width, scaled_obj_size.height)) = 1.0f; + default_mask_area = static_cast(sum(default_mask)[0]); + + //initalize segmentation + if(params.use_segmentation) { + Mat hsv_img = bgr2hsv(image); + hist_foreground = Histogram(hsv_img.channels(), params.histogram_bins); + hist_background = Histogram(hsv_img.channels(), params.histogram_bins); + extract_histograms(hsv_img, bounding_box, hist_foreground, hist_background); + filter_mask = segment_region(hsv_img, object_center, template_size, + original_target_size, current_scale_factor); + //update calculated mask with preset mask + if(preset_mask.data){ + Mat preset_mask_padded = Mat::zeros(filter_mask.size(), filter_mask.type()); + int sx = std::max((int)cvFloor(preset_mask_padded.cols / 2.0f - preset_mask.cols / 2.0f) - 1, 0); + int sy = std::max((int)cvFloor(preset_mask_padded.rows / 2.0f - preset_mask.rows / 2.0f) - 1, 0); + preset_mask.copyTo(preset_mask_padded( + Rect(sx, sy, preset_mask.cols, preset_mask.rows))); + filter_mask = filter_mask.mul(preset_mask_padded); + } + erode_element = getStructuringElement(MORPH_ELLIPSE, Size(3,3), Point(1,1)); + resize(filter_mask, filter_mask, yf.size(), 0, 0, INTER_NEAREST); + if(check_mask_area(filter_mask, default_mask_area)) { + dilate(filter_mask , filter_mask, erode_element); + } else { + filter_mask = default_mask; + } + + } else { + filter_mask = default_mask; + } + + //initialize filter + Mat patch = get_subwindow(image, object_center, cvFloor(current_scale_factor * template_size.width), + cvFloor(current_scale_factor * template_size.height)); + resize(patch, patch, rescaled_template_size, 0, 0, INTER_CUBIC); + std::vector patch_ftrs = get_features(patch, yf.size()); + std::vector Fftrs = fourier_transform_features(patch_ftrs); + csr_filter = create_csr_filter(Fftrs, yf, filter_mask); + + if(params.use_channel_weights) { + Mat current_resp; + filter_weights = std::vector(csr_filter.size()); + float chw_sum = 0; + for (size_t i = 0; i < csr_filter.size(); ++i) { + mulSpectrums(Fftrs[i], csr_filter[i], current_resp, 0, true); + idft(current_resp, current_resp, DFT_SCALE | DFT_REAL_OUTPUT); + double max_val; + minMaxLoc(current_resp, NULL, &max_val, NULL , NULL); + chw_sum += static_cast(max_val); + filter_weights[i] = static_cast(max_val); + } + for (size_t i = 0; i < filter_weights.size(); ++i) { + filter_weights[i] /= chw_sum; + } + } + + //initialize scale search + dsst = DSST(image, bounding_box, template_size, params.number_of_scales, params.scale_step, + params.scale_model_max_area, params.scale_sigma_factor, params.scale_lr); + + model=makePtr(); +} + +} // namespace impl + +TrackerCSRT::Params::Params() +{ + use_channel_weights = true; + use_segmentation = true; + use_hog = true; + use_color_names = true; + use_gray = true; + use_rgb = false; + window_function = "hann"; + kaiser_alpha = 3.75f; + cheb_attenuation = 45; + padding = 3.0f; + template_size = 200; + gsl_sigma = 1.0f; + hog_orientations = 9; + hog_clip = 0.2f; + num_hog_channels_used = 18; + filter_lr = 0.02f; + weights_lr = 0.02f; + admm_iterations = 4; + number_of_scales = 33; + scale_sigma_factor = 0.250f; + scale_model_max_area = 512.0f; + scale_lr = 0.025f; + scale_step = 1.020f; + histogram_bins = 16; + background_ratio = 2; + histogram_lr = 0.04f; + psr_threshold = 0.035f; +} + +TrackerCSRT::TrackerCSRT() +{ + // nothing +} + +TrackerCSRT::~TrackerCSRT() +{ + // nothing +} + +Ptr TrackerCSRT::create(const TrackerCSRT::Params ¶meters) +{ + return makePtr(parameters); +} + +}} // namespace + +#include "legacy/trackerCSRT.legacy.hpp" diff --git a/modules/tracking/src/trackerCSRTScaleEstimation.cpp b/modules/tracking/src/trackerCSRTScaleEstimation.cpp index 5bde5f420..221d1ee64 100644 --- a/modules/tracking/src/trackerCSRTScaleEstimation.cpp +++ b/modules/tracking/src/trackerCSRTScaleEstimation.cpp @@ -127,7 +127,7 @@ DSST::DSST(const Mat &image, mulSpectrums(ysf, Fscale_resp, sf_num, 0 , true); Mat sf_den_all; mulSpectrums(Fscale_resp, Fscale_resp, sf_den_all, 0, true); - reduce(sf_den_all, sf_den, 0, CV_REDUCE_SUM, -1); + reduce(sf_den_all, sf_den, 0, REDUCE_SUM, -1); } DSST::~DSST() @@ -178,7 +178,7 @@ void DSST::update(const Mat &image, const Point2f object_center) mulSpectrums(ysf, Fscale_features, new_sf_num, DFT_ROWS, true); Mat sf_den_all; mulSpectrums(Fscale_features, Fscale_features, new_sf_den_all, DFT_ROWS, true); - reduce(new_sf_den_all, new_sf_den, 0, CV_REDUCE_SUM, -1); + reduce(new_sf_den_all, new_sf_den, 0, REDUCE_SUM, -1); sf_num = (1 - learn_rate) * sf_num + learn_rate * new_sf_num; sf_den = (1 - learn_rate) * sf_den + learn_rate * new_sf_den; @@ -194,7 +194,7 @@ float DSST::getScale(const Mat &image, const Point2f object_center) mulSpectrums(Fscale_features, sf_num, Fscale_features, 0, false); Mat scale_resp; - reduce(Fscale_features, scale_resp, 0, CV_REDUCE_SUM, -1); + reduce(Fscale_features, scale_resp, 0, REDUCE_SUM, -1); scale_resp = divide_complex_matrices(scale_resp, sf_den + 0.01f); idft(scale_resp, scale_resp, DFT_REAL_OUTPUT|DFT_SCALE); Point max_loc; diff --git a/modules/tracking/src/trackerFeature.cpp b/modules/tracking/src/trackerFeature.cpp index a5b59bdcf..3f86fb5d0 100644 --- a/modules/tracking/src/trackerFeature.cpp +++ b/modules/tracking/src/trackerFeature.cpp @@ -41,8 +41,9 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerFeature @@ -321,4 +322,4 @@ void TrackerFeatureLBP::selection( Mat& /*response*/, int /*npoints*/) } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerFeatureSet.cpp b/modules/tracking/src/trackerFeatureSet.cpp index 9896f0ffa..dfa847a51 100644 --- a/modules/tracking/src/trackerFeatureSet.cpp +++ b/modules/tracking/src/trackerFeatureSet.cpp @@ -41,8 +41,9 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerFeatureSet @@ -139,4 +140,5 @@ void TrackerFeatureSet::clearResponses() responses.clear(); } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerKCF.cpp b/modules/tracking/src/trackerKCF.cpp new file mode 100644 index 000000000..9623f2f1b --- /dev/null +++ b/modules/tracking/src/trackerKCF.cpp @@ -0,0 +1,938 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" + +#include "opencl_kernels_tracking.hpp" +#include +#include + +namespace cv { +inline namespace tracking { +namespace impl { + +/*--------------------------- +| TrackerKCFModel +|---------------------------*/ + /** + * \brief Implementation of TrackerModel for KCF algorithm + */ + class TrackerKCFModel : public TrackerModel{ + public: + TrackerKCFModel(){} + ~TrackerKCFModel(){} + protected: + void modelEstimationImpl( const std::vector& /*responses*/ ) CV_OVERRIDE {} + void modelUpdateImpl() CV_OVERRIDE {} + }; + + +/*--------------------------- +| TrackerKCF +|---------------------------*/ + /* + * Prototype + */ +class TrackerKCFImpl CV_FINAL : public TrackerKCF +{ +public: + TrackerKCFImpl(const TrackerKCF::Params ¶meters); + + virtual void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + virtual bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + void setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func = false) CV_OVERRIDE; + + TrackerKCF::Params params; + Ptr model; + +protected: + void createHanningWindow(OutputArray dest, const cv::Size winSize, const int type) const; + void inline fft2(const Mat src, std::vector & dest, std::vector & layers_data) const; + void inline fft2(const Mat src, Mat & dest) const; + void inline ifft2(const Mat src, Mat & dest) const; + void inline pixelWiseMult(const std::vector src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB=false) const; + void inline sumChannels(std::vector src, Mat & dest) const; + void inline updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & proj_matrix,float pca_rate, int compressed_sz, + std::vector & layers_pca,std::vector & average, Mat pca_data, Mat new_cov, Mat w, Mat u, Mat v); + void inline compress(const Mat proj_matrix, const Mat src, Mat & dest, Mat & data, Mat & compressed) const; + bool getSubWindow(const Mat img, const Rect roi, Mat& feat, Mat& patch, TrackerKCF::MODE desc = GRAY) const; + bool getSubWindow(const Mat img, const Rect roi, Mat& feat, void (*f)(const Mat, const Rect, Mat& )) const; + void extractCN(Mat patch_data, Mat & cnFeatures) const; + void denseGaussKernel(const float sigma, const Mat , const Mat y_data, Mat & k_data, + std::vector & layers_data,std::vector & xf_data,std::vector & yf_data, std::vector xyf_v, Mat xy, Mat xyf ) const; + void calcResponse(const Mat alphaf_data, const Mat kf_data, Mat & response_data, Mat & spec_data) const; + void calcResponse(const Mat alphaf_data, const Mat alphaf_den_data, const Mat kf_data, Mat & response_data, Mat & spec_data, Mat & spec2_data) const; + + void shiftRows(Mat& mat) const; + void shiftRows(Mat& mat, int n) const; + void shiftCols(Mat& mat, int n) const; +#ifdef HAVE_OPENCL + bool inline oclTransposeMM(const Mat src, float alpha, UMat &dst); +#endif + +private: + float output_sigma; + Rect2d roi; + Mat hann; //hann window filter + Mat hann_cn; //10 dimensional hann-window filter for CN features, + + Mat y,yf; // training response and its FFT + Mat x; // observation and its FFT + Mat k,kf; // dense gaussian kernel and its FFT + Mat kf_lambda; // kf+lambda + Mat new_alphaf, alphaf; // training coefficients + Mat new_alphaf_den, alphaf_den; // for splitted training coefficients + Mat z; // model + Mat response; // detection result + Mat old_cov_mtx, proj_mtx; // for feature compression + + // pre-defined Mat variables for optimization of private functions + Mat spec, spec2; + std::vector layers; + std::vector vxf,vyf,vxyf; + Mat xy_data,xyf_data; + Mat data_temp, compress_data; + std::vector layers_pca_data; + std::vector average_data; + Mat img_Patch; + + // storage for the extracted features, KRLS model, KRLS compressed model + Mat X[2],Z[2],Zc[2]; + + // storage of the extracted features + std::vector features_pca; + std::vector features_npca; + std::vector descriptors_pca; + std::vector descriptors_npca; + + // optimization variables for updateProjectionMatrix + Mat data_pca, new_covar,w_data,u_data,vt_data; + + // custom feature extractor + bool use_custom_extractor_pca; + bool use_custom_extractor_npca; + std::vector extractor_pca; + std::vector extractor_npca; + + bool resizeImage; // resize the image whenever needed and the patch size is large + +#ifdef HAVE_OPENCL + ocl::Kernel transpose_mm_ker; // OCL kernel to compute transpose matrix multiply matrix. +#endif + + int frame; +}; + + /* + * Constructor + */ + TrackerKCFImpl::TrackerKCFImpl( const TrackerKCF::Params ¶meters ) : + params( parameters ) + { + resizeImage = false; + use_custom_extractor_pca = false; + use_custom_extractor_npca = false; + +#ifdef HAVE_OPENCL + // For update proj matrix's multiplication + if(ocl::useOpenCL()) + { + cv::String err; + ocl::ProgramSource tmmSrc = ocl::tracking::tmm_oclsrc; + ocl::Program tmmProg(tmmSrc, String(), err); + transpose_mm_ker.create("tmm", tmmProg); + } +#endif + } + + /* + * Initialization: + * - creating hann window filter + * - ROI padding + * - creating a gaussian response for the training ground-truth + * - perform FFT to the gaussian response + */ + void TrackerKCFImpl::init(InputArray image, const Rect& boundingBox) + { + frame=0; + roi.x = cvRound(boundingBox.x); + roi.y = cvRound(boundingBox.y); + roi.width = cvRound(boundingBox.width); + roi.height = cvRound(boundingBox.height); + + //calclulate output sigma + output_sigma=std::sqrt(static_cast(roi.width*roi.height))*params.output_sigma_factor; + output_sigma=-0.5f/(output_sigma*output_sigma); + + //resize the ROI whenever needed + if(params.resize && roi.width*roi.height>params.max_patch_size){ + resizeImage=true; + roi.x/=2.0; + roi.y/=2.0; + roi.width/=2.0; + roi.height/=2.0; + } + + // add padding to the roi + roi.x-=roi.width/2; + roi.y-=roi.height/2; + roi.width*=2; + roi.height*=2; + + // initialize the hann window filter + createHanningWindow(hann, roi.size(), CV_32F); + + // hann window filter for CN feature + Mat _layer[] = {hann, hann, hann, hann, hann, hann, hann, hann, hann, hann}; + merge(_layer, 10, hann_cn); + + // create gaussian response + y=Mat::zeros((int)roi.height,(int)roi.width,CV_32F); + for(int i=0;i(i,j) = + static_cast((i-roi.height/2+1)*(i-roi.height/2+1)+(j-roi.width/2+1)*(j-roi.width/2+1)); + } + } + + y*=(float)output_sigma; + cv::exp(y,y); + + // perform fourier transfor to the gaussian response + fft2(y,yf); + + if (image.channels() == 1) { // disable CN for grayscale images + params.desc_pca &= ~(CN); + params.desc_npca &= ~(CN); + } + model = makePtr(); + + // record the non-compressed descriptors + if((params.desc_npca & GRAY) == GRAY)descriptors_npca.push_back(GRAY); + if((params.desc_npca & CN) == CN)descriptors_npca.push_back(CN); + if(use_custom_extractor_npca)descriptors_npca.push_back(CUSTOM); + features_npca.resize(descriptors_npca.size()); + + // record the compressed descriptors + if((params.desc_pca & GRAY) == GRAY)descriptors_pca.push_back(GRAY); + if((params.desc_pca & CN) == CN)descriptors_pca.push_back(CN); + if(use_custom_extractor_pca)descriptors_pca.push_back(CUSTOM); + features_pca.resize(descriptors_pca.size()); + + // accept only the available descriptor modes + CV_Assert( + (params.desc_pca & GRAY) == GRAY + || (params.desc_npca & GRAY) == GRAY + || (params.desc_pca & CN) == CN + || (params.desc_npca & CN) == CN + || use_custom_extractor_pca + || use_custom_extractor_npca + ); + + // ensure roi has intersection with the image + Rect2d image_roi(0, 0, + image.cols() / (resizeImage ? 2 : 1), + image.rows() / (resizeImage ? 2 : 1)); + CV_Assert(!(roi & image_roi).empty()); + } + + /* + * Main part of the KCF algorithm + */ + bool TrackerKCFImpl::update(InputArray image, Rect& boundingBoxResult) + { + double minVal, maxVal; // min-max response + Point minLoc,maxLoc; // min-max location + + CV_Assert(image.channels() == 1 || image.channels() == 3); + + Mat img; + // resize the image whenever needed + if (resizeImage) + resize(image, img, Size(image.cols()/2, image.rows()/2), 0, 0, INTER_LINEAR_EXACT); + else + image.copyTo(img); + + // detection part + if(frame>0){ + + // extract and pre-process the patch + // get non compressed descriptors + for(unsigned i=0;i0)merge(features_npca,X[1]); + + // get compressed descriptors + for(unsigned i=0;i0)merge(features_pca,X[0]); + + //compress the features and the KRSL model + if(params.desc_pca !=0){ + compress(proj_mtx,X[0],X[0],data_temp,compress_data); + compress(proj_mtx,Z[0],Zc[0],data_temp,compress_data); + } + + // copy the compressed KRLS model + Zc[1] = Z[1]; + + // merge all features + if(features_npca.size()==0){ + x = X[0]; + z = Zc[0]; + }else if(features_pca.size()==0){ + x = X[1]; + z = Z[1]; + }else{ + merge(X,2,x); + merge(Zc,2,z); + } + + //compute the gaussian kernel + denseGaussKernel(params.sigma,x,z,k,layers,vxf,vyf,vxyf,xy_data,xyf_data); + + // compute the fourier transform of the kernel + fft2(k,kf); + if(frame==1)spec2=Mat_(kf.rows, kf.cols); + + // calculate filter response + if(params.split_coeff) + calcResponse(alphaf,alphaf_den,kf,response, spec, spec2); + else + calcResponse(alphaf,kf,response, spec); + + // extract the maximum response + minMaxLoc( response, &minVal, &maxVal, &minLoc, &maxLoc ); + if (maxVal < params.detect_thresh) + { + return false; + } + roi.x+=(maxLoc.x-roi.width/2+1); + roi.y+=(maxLoc.y-roi.height/2+1); + } + + // update the bounding box + Rect2d boundingBox; + boundingBox.x=(resizeImage?roi.x*2:roi.x)+(resizeImage?roi.width*2:roi.width)/4; + boundingBox.y=(resizeImage?roi.y*2:roi.y)+(resizeImage?roi.height*2:roi.height)/4; + boundingBox.width = (resizeImage?roi.width*2:roi.width)/2; + boundingBox.height = (resizeImage?roi.height*2:roi.height)/2; + + // extract the patch for learning purpose + // get non compressed descriptors + for(unsigned i=0;i0)merge(features_npca,X[1]); + + // get compressed descriptors + for(unsigned i=0;i0)merge(features_pca,X[0]); + + //update the training data + if(frame==0){ + Z[0] = X[0].clone(); + Z[1] = X[1].clone(); + }else{ + Z[0]=(1.0-params.interp_factor)*Z[0]+params.interp_factor*X[0]; + Z[1]=(1.0-params.interp_factor)*Z[1]+params.interp_factor*X[1]; + } + + if(params.desc_pca !=0 || use_custom_extractor_pca){ + // initialize the vector of Mat variables + if(frame==0){ + layers_pca_data.resize(Z[0].channels()); + average_data.resize(Z[0].channels()); + } + + // feature compression + updateProjectionMatrix(Z[0],old_cov_mtx,proj_mtx,params.pca_learning_rate,params.compressed_size,layers_pca_data,average_data,data_pca, new_covar,w_data,u_data,vt_data); + compress(proj_mtx,X[0],X[0],data_temp,compress_data); + } + + // merge all features + if(features_npca.size()==0) + x = X[0]; + else if(features_pca.size()==0) + x = X[1]; + else + merge(X,2,x); + + // initialize some required Mat variables + if(frame==0){ + layers.resize(x.channels()); + vxf.resize(x.channels()); + vyf.resize(x.channels()); + vxyf.resize(vyf.size()); + new_alphaf=Mat_(yf.rows, yf.cols); + } + + // Kernel Regularized Least-Squares, calculate alphas + denseGaussKernel(params.sigma,x,x,k,layers,vxf,vyf,vxyf,xy_data,xyf_data); + + // compute the fourier transform of the kernel and add a small value + fft2(k,kf); + kf_lambda=kf+params.lambda; + + float den; + if(params.split_coeff){ + mulSpectrums(yf,kf,new_alphaf,0); + mulSpectrums(kf,kf_lambda,new_alphaf_den,0); + }else{ + for(int i=0;i(i,j)[0]*kf_lambda.at(i,j)[0]+kf_lambda.at(i,j)[1]*kf_lambda.at(i,j)[1]); + + new_alphaf.at(i,j)[0]= + (yf.at(i,j)[0]*kf_lambda.at(i,j)[0]+yf.at(i,j)[1]*kf_lambda.at(i,j)[1])*den; + new_alphaf.at(i,j)[1]= + (yf.at(i,j)[1]*kf_lambda.at(i,j)[0]-yf.at(i,j)[0]*kf_lambda.at(i,j)[1])*den; + } + } + } + + // update the RLS model + if(frame==0){ + alphaf=new_alphaf.clone(); + if(params.split_coeff)alphaf_den=new_alphaf_den.clone(); + }else{ + alphaf=(1.0-params.interp_factor)*alphaf+params.interp_factor*new_alphaf; + if(params.split_coeff)alphaf_den=(1.0-params.interp_factor)*alphaf_den+params.interp_factor*new_alphaf_den; + } + + frame++; + + int x1 = cvRound(boundingBox.x); + int y1 = cvRound(boundingBox.y); + int x2 = cvRound(boundingBox.x + boundingBox.width); + int y2 = cvRound(boundingBox.y + boundingBox.height); + boundingBoxResult = Rect(x1, y1, x2 - x1, y2 - y1) & Rect(Point(0, 0), image.size()); + + return true; + } + + + /*------------------------------------- + | implementation of the KCF functions + |-------------------------------------*/ + + /* + * hann window filter + */ + void TrackerKCFImpl::createHanningWindow(OutputArray dest, const cv::Size winSize, const int type) const { + CV_Assert( type == CV_32FC1 || type == CV_64FC1 ); + + dest.create(winSize, type); + Mat dst = dest.getMat(); + + int rows = dst.rows, cols = dst.cols; + + AutoBuffer _wc(cols); + float * const wc = _wc.data(); + + const float coeff0 = 2.0f * (float)CV_PI / (cols - 1); + const float coeff1 = 2.0f * (float)CV_PI / (rows - 1); + for(int j = 0; j < cols; j++) + wc[j] = 0.5f * (1.0f - cos(coeff0 * j)); + + if(dst.depth() == CV_32F){ + for(int i = 0; i < rows; i++){ + float* dstData = dst.ptr(i); + float wr = 0.5f * (1.0f - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = (float)(wr * wc[j]); + } + }else{ + for(int i = 0; i < rows; i++){ + double* dstData = dst.ptr(i); + double wr = 0.5f * (1.0f - cos(coeff1 * i)); + for(int j = 0; j < cols; j++) + dstData[j] = wr * wc[j]; + } + } + + // perform batch sqrt for SSE performance gains + //cv::sqrt(dst, dst); //matlab do not use the square rooted version + } + + /* + * simplification of fourier transform function in opencv + */ + void inline TrackerKCFImpl::fft2(const Mat src, Mat & dest) const { + dft(src,dest,DFT_COMPLEX_OUTPUT); + } + + void inline TrackerKCFImpl::fft2(const Mat src, std::vector & dest, std::vector & layers_data) const { + split(src, layers_data); + + for(int i=0;i src1, const std::vector src2, std::vector & dest, const int flags, const bool conjB) const { + for(unsigned i=0;i src, Mat & dest) const { + dest=src[0].clone(); + for(unsigned i=1;i(src.cols * 64), static_cast(src.cols)}; + size_t localSize[2] = {64, 1}; + if (!transpose_mm_ker.run(2, globSize, localSize, true)) + return false; + return true; + } +#endif + + /* + * obtains the projection matrix using PCA + */ + void inline TrackerKCFImpl::updateProjectionMatrix(const Mat src, Mat & old_cov,Mat & proj_matrix, float pca_rate, int compressed_sz, + std::vector & layers_pca,std::vector & average, Mat pca_data, Mat new_cov, Mat w, Mat u, Mat vt) { + CV_Assert(compressed_sz<=src.channels()); + + split(src,layers_pca); + + for (int i=0;i(i, j) - result.getMat(ACCESS_RW).at(i , j)) > abs(new_cov.at(i, j)) * 1e-3) + printf("error @ i %d j %d got %G expected %G \n", i, j, result.getMat(ACCESS_RW).at(i , j), new_cov.at(i, j)); +#endif + if(old_cov.rows==0)old_cov=new_cov.clone(); + SVD::compute((1.0f - pca_rate) * old_cov + pca_rate * new_cov, w, u, vt); + } +#else + new_cov=1.0/(float)(src.rows*src.cols-1)*(pca_data.t()*pca_data); + if(old_cov.rows==0)old_cov=new_cov.clone(); + + // calc PCA + SVD::compute((1.0-pca_rate)*old_cov+pca_rate*new_cov, w, u, vt); +#endif + // extract the projection matrix + proj_matrix=u(Rect(0,0,compressed_sz,src.channels())).clone(); + Mat proj_vars=Mat::eye(compressed_sz,compressed_sz,proj_matrix.type()); + for(int i=0;i(i,i)=w.at(i); + } + + // update the covariance matrix + old_cov=(1.0-pca_rate)*old_cov+pca_rate*proj_matrix*proj_vars*proj_matrix.t(); + } + + /* + * compress the features + */ + void inline TrackerKCFImpl::compress(const Mat proj_matrix, const Mat src, Mat & dest, Mat & data, Mat & compressed) const { + data=src.reshape(1,src.rows*src.cols); + compressed=data*proj_matrix; + dest=compressed.reshape(proj_matrix.cols,src.rows).clone(); + } + + /* + * obtain the patch and apply hann window filter to it + */ + bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& feat, Mat& patch, TrackerKCF::MODE desc) const { + + Rect region=_roi; + + // return false if roi is outside the image + if ((roi & Rect2d(0, 0, img.cols, img.rows)).empty()) + return false; + + // extract patch inside the image + if(_roi.x<0){region.x=0;region.width+=_roi.x;} + if(_roi.y<0){region.y=0;region.height+=_roi.y;} + if(_roi.x+_roi.width>img.cols)region.width=img.cols-_roi.x; + if(_roi.y+_roi.height>img.rows)region.height=img.rows-_roi.y; + if(region.width>img.cols)region.width=img.cols; + if(region.height>img.rows)region.height=img.rows; + + // return false if region is empty + if (region.empty()) + return false; + + patch=img(region).clone(); + + // add some padding to compensate when the patch is outside image border + int addTop,addBottom, addLeft, addRight; + addTop=region.y-_roi.y; + addBottom=(_roi.height+_roi.y>img.rows?_roi.height+_roi.y-img.rows:0); + addLeft=region.x-_roi.x; + addRight=(_roi.width+_roi.x>img.cols?_roi.width+_roi.x-img.cols:0); + + copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE); + if(patch.rows==0 || patch.cols==0)return false; + + // extract the desired descriptors + switch(desc){ + case CN: + CV_Assert(img.channels() == 3); + extractCN(patch,feat); + feat=feat.mul(hann_cn); // hann window filter + break; + default: // GRAY + if(img.channels()>1) + cvtColor(patch,feat, COLOR_BGR2GRAY); + else + feat=patch; + //feat.convertTo(feat,CV_32F); + feat.convertTo(feat,CV_32F, 1.0/255.0, -0.5); + //feat=feat/255.0-0.5; // normalize to range -0.5 .. 0.5 + feat=feat.mul(hann); // hann window filter + break; + } + + return true; + + } + + /* + * get feature using external function + */ + bool TrackerKCFImpl::getSubWindow(const Mat img, const Rect _roi, Mat& feat, void (*f)(const Mat, const Rect, Mat& )) const{ + + // return false if roi is outside the image + if((_roi.x+_roi.width<0) + ||(_roi.y+_roi.height<0) + ||(_roi.x>=img.cols) + ||(_roi.y>=img.rows) + )return false; + + f(img, _roi, feat); + + if(_roi.width != feat.cols || _roi.height != feat.rows){ + printf("error in customized function of features extractor!\n"); + printf("Rules: roi.width==feat.cols && roi.height = feat.rows \n"); + } + + Mat hann_win; + std::vector _layers; + + for(int i=0;i(0,0); + unsigned index; + + if(cnFeatures.type() != CV_32FC(10)) + cnFeatures = Mat::zeros(patch_data.rows,patch_data.cols,CV_32FC(10)); + + for(int i=0;i(i,j); + index=(unsigned)(floor((float)pixel[2]/8)+32*floor((float)pixel[1]/8)+32*32*floor((float)pixel[0]/8)); + + //copy the values + for(int _k=0;_k<10;_k++){ + cnFeatures.at >(i,j)[_k]=ColorNames[index][_k]; + } + } + } + + } + + /* + * dense gauss kernel function + */ + void TrackerKCFImpl::denseGaussKernel(const float sigma, const Mat x_data, const Mat y_data, Mat & k_data, + std::vector & layers_data,std::vector & xf_data,std::vector & yf_data, std::vector xyf_v, Mat xy, Mat xyf ) const { + double normX, normY; + + fft2(x_data,xf_data,layers_data); + fft2(y_data,yf_data,layers_data); + + normX=norm(x_data); + normX*=normX; + normY=norm(y_data); + normY*=normY; + + pixelWiseMult(xf_data,yf_data,xyf_v,0,true); + sumChannels(xyf_v,xyf); + ifft2(xyf,xyf); + + if(params.wrap_kernel){ + shiftRows(xyf, x_data.rows/2); + shiftCols(xyf, x_data.cols/2); + } + + //(xx + yy - 2 * xy) / numel(x) + xy=(normX+normY-2*xyf)/(x_data.rows*x_data.cols*x_data.channels()); + + // TODO: check wether we really need thresholding or not + //threshold(xy,xy,0.0,0.0,THRESH_TOZERO);//max(0, (xx + yy - 2 * xy) / numel(x)) + for(int i=0;i(i,j)<0.0)xy.at(i,j)=0.0; + } + } + + float sig=-1.0f/(sigma*sigma); + xy=sig*xy; + exp(xy,k_data); + + } + + /* CIRCULAR SHIFT Function + * http://stackoverflow.com/questions/10420454/shift-like-matlab-function-rows-or-columns-of-a-matrix-in-opencv + */ + // circular shift one row from up to down + void TrackerKCFImpl::shiftRows(Mat& mat) const { + + Mat temp; + Mat m; + int _k = (mat.rows-1); + mat.row(_k).copyTo(temp); + for(; _k > 0 ; _k-- ) { + m = mat.row(_k); + mat.row(_k-1).copyTo(m); + } + m = mat.row(0); + temp.copyTo(m); + + } + + // circular shift n rows from up to down if n > 0, -n rows from down to up if n < 0 + void TrackerKCFImpl::shiftRows(Mat& mat, int n) const { + if( n < 0 ) { + n = -n; + flip(mat,mat,0); + for(int _k=0; _k < n;_k++) { + shiftRows(mat); + } + flip(mat,mat,0); + }else{ + for(int _k=0; _k < n;_k++) { + shiftRows(mat); + } + } + } + + //circular shift n columns from left to right if n > 0, -n columns from right to left if n < 0 + void TrackerKCFImpl::shiftCols(Mat& mat, int n) const { + if(n < 0){ + n = -n; + flip(mat,mat,1); + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); + flip(mat,mat,1); + }else{ + transpose(mat,mat); + shiftRows(mat,n); + transpose(mat,mat); + } + } + + /* + * calculate the detection response + */ + void TrackerKCFImpl::calcResponse(const Mat alphaf_data, const Mat kf_data, Mat & response_data, Mat & spec_data) const { + //alpha f--> 2channels ; k --> 1 channel; + mulSpectrums(alphaf_data,kf_data,spec_data,0,false); + ifft2(spec_data,response_data); + } + + /* + * calculate the detection response for splitted form + */ + void TrackerKCFImpl::calcResponse(const Mat alphaf_data, const Mat _alphaf_den, const Mat kf_data, Mat & response_data, Mat & spec_data, Mat & spec2_data) const { + + mulSpectrums(alphaf_data,kf_data,spec_data,0,false); + + //z=(a+bi)/(c+di)=[(ac+bd)+i(bc-ad)]/(c^2+d^2) + float den; + for(int i=0;i(i,j)[0]*_alphaf_den.at(i,j)[0]+_alphaf_den.at(i,j)[1]*_alphaf_den.at(i,j)[1]); + spec2_data.at(i,j)[0]= + (spec_data.at(i,j)[0]*_alphaf_den.at(i,j)[0]+spec_data.at(i,j)[1]*_alphaf_den.at(i,j)[1])*den; + spec2_data.at(i,j)[1]= + (spec_data.at(i,j)[1]*_alphaf_den.at(i,j)[0]-spec_data.at(i,j)[0]*_alphaf_den.at(i,j)[1])*den; + } + } + + ifft2(spec2_data,response_data); + } + + void TrackerKCFImpl::setFeatureExtractor(void (*f)(const Mat, const Rect, Mat&), bool pca_func){ + if(pca_func){ + extractor_pca.push_back(f); + use_custom_extractor_pca = true; + }else{ + extractor_npca.push_back(f); + use_custom_extractor_npca = true; + } + } + /*----------------------------------------------------------------------*/ + + +} // namespace + +TrackerKCF::Params::Params() +{ + detect_thresh = 0.5f; + sigma=0.2f; + lambda=0.0001f; + interp_factor=0.075f; + output_sigma_factor=1.0f / 16.0f; + resize=true; + max_patch_size=80*80; + split_coeff=true; + wrap_kernel=false; + desc_npca = GRAY; + desc_pca = CN; + + //feature compression + compress_feature=true; + compressed_size=2; + pca_learning_rate=0.15f; +} + + +TrackerKCF::TrackerKCF() +{ + // nothing +} + +TrackerKCF::~TrackerKCF() +{ + // nothing +} + +Ptr TrackerKCF::create(const TrackerKCF::Params ¶meters) +{ + return makePtr(parameters); +} + +}} // namespace + +#include "legacy/trackerKCF.legacy.hpp" diff --git a/modules/tracking/src/trackerMIL.cpp b/modules/tracking/src/trackerMIL.cpp new file mode 100644 index 000000000..3d3c22c1f --- /dev/null +++ b/modules/tracking/src/trackerMIL.cpp @@ -0,0 +1,265 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" +#include "trackerMILModel.hpp" + +namespace cv { +inline namespace tracking { +namespace impl { + +class TrackerMILImpl CV_FINAL : public TrackerMIL +{ +public: + TrackerMILImpl(const TrackerMIL::Params ¶meters); + + virtual void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + virtual bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + + + void compute_integral( const Mat & img, Mat & ii_img ); + + TrackerMIL::Params params; + + Ptr model; + Ptr sampler; + Ptr featureSet; +}; + +} // namespace + +TrackerMIL::Params::Params() +{ + samplerInitInRadius = 3; + samplerSearchWinSize = 25; + samplerInitMaxNegNum = 65; + samplerTrackInRadius = 4; + samplerTrackMaxPosNum = 100000; + samplerTrackMaxNegNum = 65; + featureSetNumFeatures = 250; +} + +namespace impl { + +TrackerMILImpl::TrackerMILImpl(const TrackerMIL::Params ¶meters) + : params(parameters) +{ + // nothing +} + +void TrackerMILImpl::compute_integral( const Mat & img, Mat & ii_img ) +{ + Mat ii; + std::vector ii_imgs; + integral( img, ii, CV_32F ); // FIXIT split first + split( ii, ii_imgs ); + ii_img = ii_imgs[0]; +} + +void TrackerMILImpl::init(InputArray image, const Rect& boundingBox) +{ + sampler = makePtr(); + featureSet = makePtr(); + + Mat intImage; + compute_integral(image.getMat(), intImage); + TrackerSamplerCSC::Params CSCparameters; + CSCparameters.initInRad = params.samplerInitInRadius; + CSCparameters.searchWinSize = params.samplerSearchWinSize; + CSCparameters.initMaxNegNum = params.samplerInitMaxNegNum; + CSCparameters.trackInPosRad = params.samplerTrackInRadius; + CSCparameters.trackMaxPosNum = params.samplerTrackMaxPosNum; + CSCparameters.trackMaxNegNum = params.samplerTrackMaxNegNum; + + Ptr CSCSampler = makePtr(CSCparameters); + CV_Assert(sampler->addTrackerSamplerAlgorithm(CSCSampler)); + + //or add CSC sampler with default parameters + //sampler->addTrackerSamplerAlgorithm( "CSC" ); + + //Positive sampling + CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); + sampler->sampling( intImage, boundingBox ); + std::vector posSamples = sampler->getSamples(); + + //Negative sampling + CSCSampler.staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); + sampler->sampling( intImage, boundingBox ); + std::vector negSamples = sampler->getSamples(); + + CV_Assert(!posSamples.empty()); + CV_Assert(!negSamples.empty()); + + //compute HAAR features + TrackerFeatureHAAR::Params HAARparameters; + HAARparameters.numFeatures = params.featureSetNumFeatures; + HAARparameters.rectSize = Size( (int)boundingBox.width, (int)boundingBox.height ); + HAARparameters.isIntegral = true; + Ptr trackerFeature = Ptr( new TrackerFeatureHAAR( HAARparameters ) ); + featureSet->addTrackerFeature( trackerFeature ); + + featureSet->extraction( posSamples ); + const std::vector posResponse = featureSet->getResponses(); + + featureSet->extraction( negSamples ); + const std::vector negResponse = featureSet->getResponses(); + + model = makePtr(boundingBox); + Ptr stateEstimator = Ptr( + new TrackerStateEstimatorMILBoosting( params.featureSetNumFeatures ) ); + model->setTrackerStateEstimator( stateEstimator ); + + //Run model estimation and update + model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); + model->modelEstimation( posResponse ); + model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); + model->modelEstimation( negResponse ); + model->modelUpdate(); +} + +bool TrackerMILImpl::update(InputArray image, Rect& boundingBox) +{ + Mat intImage; + compute_integral(image.getMat(), intImage); + + //get the last location [AAM] X(k-1) + Ptr lastLocation = model->getLastTargetState(); + Rect lastBoundingBox( (int)lastLocation->getTargetPosition().x, (int)lastLocation->getTargetPosition().y, lastLocation->getTargetWidth(), + lastLocation->getTargetHeight() ); + + //sampling new frame based on last location + ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_DETECT ); + sampler->sampling( intImage, lastBoundingBox ); + std::vector detectSamples = sampler->getSamples(); + if( detectSamples.empty() ) + return false; + + /*//TODO debug samples + Mat f; + image.copyTo(f); + + for( size_t i = 0; i < detectSamples.size(); i=i+10 ) + { + Size sz; + Point off; + detectSamples.at(i).locateROI(sz, off); + rectangle(f, Rect(off.x,off.y,detectSamples.at(i).cols,detectSamples.at(i).rows), Scalar(255,0,0), 1); + }*/ + + //extract features from new samples + featureSet->extraction( detectSamples ); + std::vector response = featureSet->getResponses(); + + //predict new location + ConfidenceMap cmap; + model.staticCast()->setMode( TrackerMILModel::MODE_ESTIMATON, detectSamples ); + model.staticCast()->responseToConfidenceMap( response, cmap ); + model->getTrackerStateEstimator().staticCast()->setCurrentConfidenceMap( cmap ); + + if( !model->runStateEstimator() ) + { + return false; + } + + Ptr currentState = model->getLastTargetState(); + boundingBox = Rect( (int)currentState->getTargetPosition().x, (int)currentState->getTargetPosition().y, currentState->getTargetWidth(), + currentState->getTargetHeight() ); + + /*//TODO debug + rectangle(f, lastBoundingBox, Scalar(0,255,0), 1); + rectangle(f, boundingBox, Scalar(0,0,255), 1); + imshow("f", f); + //waitKey( 0 );*/ + + //sampling new frame based on new location + //Positive sampling + ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_POS ); + sampler->sampling( intImage, boundingBox ); + std::vector posSamples = sampler->getSamples(); + + //Negative sampling + ( sampler->getSamplers().at( 0 ).second ).staticCast()->setMode( TrackerSamplerCSC::MODE_INIT_NEG ); + sampler->sampling( intImage, boundingBox ); + std::vector negSamples = sampler->getSamples(); + + if( posSamples.empty() || negSamples.empty() ) + return false; + + //extract features + featureSet->extraction( posSamples ); + std::vector posResponse = featureSet->getResponses(); + + featureSet->extraction( negSamples ); + std::vector negResponse = featureSet->getResponses(); + + //model estimate + model.staticCast()->setMode( TrackerMILModel::MODE_POSITIVE, posSamples ); + model->modelEstimation( posResponse ); + model.staticCast()->setMode( TrackerMILModel::MODE_NEGATIVE, negSamples ); + model->modelEstimation( negResponse ); + + //model update + model->modelUpdate(); + + return true; +} + +} // namespace + + +TrackerMIL::TrackerMIL() +{ + // nothing +} + +TrackerMIL::~TrackerMIL() +{ + // nothing +} + +Ptr TrackerMIL::create(const TrackerMIL::Params ¶meters) +{ + return makePtr(parameters); +} + +}} // namespace + +#include "legacy/trackerMIL.legacy.hpp" diff --git a/modules/tracking/src/trackerMILModel.cpp b/modules/tracking/src/trackerMILModel.cpp index f6abec68f..0d7928904 100644 --- a/modules/tracking/src/trackerMILModel.cpp +++ b/modules/tracking/src/trackerMILModel.cpp @@ -46,8 +46,9 @@ * TrackerMILModel */ -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { TrackerMILModel::TrackerMILModel( const Rect& boundingBox ) { @@ -122,4 +123,4 @@ void TrackerMILModel::setMode( int trainingMode, const std::vector& samples mode = trainingMode; } -} +}}} // namespace diff --git a/modules/tracking/src/trackerMILModel.hpp b/modules/tracking/src/trackerMILModel.hpp index cdbf9a3cb..41945be7a 100644 --- a/modules/tracking/src/trackerMILModel.hpp +++ b/modules/tracking/src/trackerMILModel.hpp @@ -42,10 +42,9 @@ #ifndef __OPENCV_TRACKER_MIL_MODEL_HPP__ #define __OPENCV_TRACKER_MIL_MODEL_HPP__ -#include "opencv2/core.hpp" - -namespace cv -{ +namespace cv { +inline namespace tracking { +namespace impl { /** * \brief Implementation of TrackerModel for MIL algorithm @@ -98,6 +97,6 @@ class TrackerMILModel : public TrackerModel int height; //initial height of the boundingBox }; -} /* namespace cv */ +}}} // namespace #endif diff --git a/modules/tracking/src/trackerMedianFlow.cpp b/modules/tracking/src/trackerMedianFlow.cpp index 6f41c3ebe..f183448e2 100644 --- a/modules/tracking/src/trackerMedianFlow.cpp +++ b/modules/tracking/src/trackerMedianFlow.cpp @@ -40,15 +40,15 @@ //M*/ #include "precomp.hpp" -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc.hpp" +#include "opencv2/tracking/tracking_legacy.hpp" + #include "tracking_utils.hpp" #include #include -namespace -{ -using namespace cv; +namespace cv { +inline namespace tracking { +namespace impl { #undef MEDIAN_FLOW_TRACKER_DEBUG_LOGS #ifdef MEDIAN_FLOW_TRACKER_DEBUG_LOGS @@ -72,7 +72,8 @@ using namespace cv; * optimize (allocation<-->reallocation) */ -class TrackerMedianFlowImpl : public TrackerMedianFlow{ +class TrackerMedianFlowImpl : public legacy::TrackerMedianFlow +{ public: TrackerMedianFlowImpl(TrackerMedianFlow::Params paramsIn = TrackerMedianFlow::Params()) {params=paramsIn;isInit=false;} void read( const FileNode& fn ) CV_OVERRIDE; @@ -95,6 +96,7 @@ private: TrackerMedianFlow::Params params; }; +static Mat getPatch(Mat image, Size patch_size, Point2f patch_center) { Mat patch; @@ -119,7 +121,7 @@ Mat getPatch(Mat image, Size patch_size, Point2f patch_center) class TrackerMedianFlowModel : public TrackerModel{ public: - TrackerMedianFlowModel(TrackerMedianFlow::Params /*params*/){} + TrackerMedianFlowModel(legacy::TrackerMedianFlow::Params /*params*/){} Rect2d getBoundingBox(){return boundingBox_;} void setBoudingBox(Rect2d boundingBox){boundingBox_=boundingBox;} Mat getImage(){return image_;} @@ -391,10 +393,11 @@ void TrackerMedianFlowImpl::check_NCC(const Mat& oldImage,const Mat& newImage, } } -} /* anonymous namespace */ +}} // namespace + +namespace legacy { +inline namespace tracking { -namespace cv -{ /* * Parameters */ @@ -442,11 +445,13 @@ void TrackerMedianFlow::Params::write( cv::FileStorage& fs ) const{ fs << "maxMedianLengthOfDisplacementDifference" << maxMedianLengthOfDisplacementDifference; } -Ptr TrackerMedianFlow::create(const TrackerMedianFlow::Params ¶meters){ - return Ptr(new TrackerMedianFlowImpl(parameters)); +Ptr TrackerMedianFlow::create(const TrackerMedianFlow::Params ¶meters) +{ + return makePtr(parameters); } -Ptr TrackerMedianFlow::create(){ - return Ptr(new TrackerMedianFlowImpl()); +Ptr TrackerMedianFlow::create() +{ + return create(TrackerMedianFlow::Params()); } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerModel.cpp b/modules/tracking/src/trackerModel.cpp index 023b3b67d..065721f3f 100644 --- a/modules/tracking/src/trackerModel.cpp +++ b/modules/tracking/src/trackerModel.cpp @@ -41,8 +41,9 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerModel @@ -174,4 +175,4 @@ void TrackerTargetState::setTargetHeight( int height ) targetHeight = height; } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/trackerSampler.cpp b/modules/tracking/src/trackerSampler.cpp index 38ea52317..28f5d0f04 100644 --- a/modules/tracking/src/trackerSampler.cpp +++ b/modules/tracking/src/trackerSampler.cpp @@ -41,8 +41,9 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerSampler @@ -139,4 +140,5 @@ void TrackerSampler::clearSamples() samples.clear(); } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerSamplerAlgorithm.cpp b/modules/tracking/src/trackerSamplerAlgorithm.cpp index ea52070d5..39d1271ba 100644 --- a/modules/tracking/src/trackerSamplerAlgorithm.cpp +++ b/modules/tracking/src/trackerSamplerAlgorithm.cpp @@ -40,18 +40,12 @@ //M*/ #include "precomp.hpp" -#include #include "PFSolver.hpp" #include "TrackingFunctionPF.hpp" -#ifdef _WIN32 -#define TIME( arg ) (((double) clock()) / CLOCKS_PER_SEC) -#else -#define TIME( arg ) (time( arg )) -#endif - -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerSamplerAlgorithm @@ -114,7 +108,7 @@ TrackerSamplerCSC::TrackerSamplerCSC( const TrackerSamplerCSC::Params ¶meter { className = "CSC"; mode = MODE_INIT_POS; - rng = RNG( uint64( TIME( 0 ) ) ); + rng = theRNG(); } @@ -410,4 +404,5 @@ bool TrackerSamplerPF::samplingImpl( const Mat& image, Rect boundingBox, std::ve return true; } -} /* namespace cv */ + +}}} // namespace diff --git a/modules/tracking/src/trackerStateEstimator.cpp b/modules/tracking/src/trackerStateEstimator.cpp index 094aef059..d0f9dd36d 100644 --- a/modules/tracking/src/trackerStateEstimator.cpp +++ b/modules/tracking/src/trackerStateEstimator.cpp @@ -41,8 +41,9 @@ #include "precomp.hpp" -namespace cv -{ +namespace cv { +namespace detail { +inline namespace tracking { /* * TrackerStateEstimator @@ -441,4 +442,4 @@ void TrackerStateEstimatorSVM::updateImpl( std::vector& /*confide } -} /* namespace cv */ +}}} // namespace diff --git a/modules/tracking/src/tracking_by_matching.cpp b/modules/tracking/src/tracking_by_matching.cpp index 3e2199b5c..8b0228306 100644 --- a/modules/tracking/src/tracking_by_matching.cpp +++ b/modules/tracking/src/tracking_by_matching.cpp @@ -2,6 +2,8 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" + #include #include #include @@ -25,7 +27,11 @@ #define TBM_CHECK_LE(actual, expected) CV_CheckLE(actual, expected, "Assertion error:") #define TBM_CHECK_GE(actual, expected) CV_CheckGE(actual, expected, "Assertion error:") -using namespace cv::tbm; +namespace cv { +namespace detail { +inline namespace tracking { + +using namespace tbm; CosDistance::CosDistance(const cv::Size &descriptor_size) : descriptor_size_(descriptor_size) { @@ -1335,3 +1341,5 @@ void TrackerByMatching::PrintConfusionMatrices() const { std::cout << cm << std::endl << std::endl; } } + +}}} // namespace diff --git a/modules/tracking/src/tracking_utils.cpp b/modules/tracking/src/tracking_utils.cpp index fd6ae1bc5..c197f902d 100644 --- a/modules/tracking/src/tracking_utils.cpp +++ b/modules/tracking/src/tracking_utils.cpp @@ -2,9 +2,10 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. +#include "precomp.hpp" #include "tracking_utils.hpp" -using namespace cv; +namespace cv { double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) { @@ -67,3 +68,5 @@ double tracking_internal::computeNCC(const Mat& patch1, const Mat& patch2) return (sq2 == 0) ? sq1 / abs(sq1) : (prod - s1 * s2 / N) / sq1 / sq2; } } + +} // namespace diff --git a/modules/tracking/src/tracking_utils.hpp b/modules/tracking/src/tracking_utils.hpp index a73562242..903b783b8 100644 --- a/modules/tracking/src/tracking_utils.hpp +++ b/modules/tracking/src/tracking_utils.hpp @@ -3,12 +3,11 @@ // of this distribution and at http://opencv.org/license.html. #ifndef __OPENCV_TRACKING_UTILS_HPP__ -#include "precomp.hpp" #include namespace cv { -namespace tracking_internal -{ +namespace tracking_internal { + /** Computes normalized corellation coefficient between the two patches (they should be * of the same size).*/ double computeNCC(const Mat& patch1, const Mat& patch2); @@ -42,6 +41,6 @@ namespace tracking_internal std::vector copy(values); return getMedianAndDoPartition(copy); } -} -} + +}} // namespace #endif diff --git a/modules/tracking/src/unscented_kalman.cpp b/modules/tracking/src/unscented_kalman.cpp index 9b8001604..d6d82a7f3 100644 --- a/modules/tracking/src/unscented_kalman.cpp +++ b/modules/tracking/src/unscented_kalman.cpp @@ -42,10 +42,10 @@ #include "precomp.hpp" #include "opencv2/tracking/kalman_filters.hpp" -namespace cv -{ -namespace tracking -{ +namespace cv { +namespace detail { +inline namespace tracking { +inline namespace kalman_filters { void UnscentedKalmanFilterParams:: init( int dp, int mp, int cp, double processNoiseCovDiag, double measurementNoiseCovDiag, @@ -369,5 +369,4 @@ Ptr createUnscentedKalmanFilter(const UnscentedKalmanFilt return kfu; } -} // tracking -} // cv +}}}} // namespace diff --git a/modules/tracking/test/test_aukf.cpp b/modules/tracking/test/test_aukf.cpp index 73372971c..32e32c96e 100644 --- a/modules/tracking/test/test_aukf.cpp +++ b/modules/tracking/test/test_aukf.cpp @@ -43,7 +43,7 @@ #include "opencv2/tracking/kalman_filters.hpp" namespace opencv_test { namespace { -using namespace cv::tracking; +using namespace cv::detail; // In this two tests Augmented Unscented Kalman Filter are applied to the dynamic system from example "The reentry problem" from // "A New Extension of the Kalman Filter to Nonlinear Systems" by Simon J. Julier and Jeffrey K. Uhlmann. diff --git a/modules/tracking/test/test_trackerParametersIO.cpp b/modules/tracking/test/test_trackerParametersIO.cpp index ebe43a5fb..59e7917e2 100644 --- a/modules/tracking/test/test_trackerParametersIO.cpp +++ b/modules/tracking/test/test_trackerParametersIO.cpp @@ -4,11 +4,15 @@ #include "test_precomp.hpp" +#include +//using namespace cv::tracking::legacy; + namespace opencv_test { namespace { + TEST(MEDIAN_FLOW_Parameters, IO) { - TrackerMedianFlow::Params parameters; + legacy::TrackerMedianFlow::Params parameters; parameters.maxLevel = 10; parameters.maxMedianLengthOfDisplacementDifference = 11; @@ -25,7 +29,7 @@ TEST(MEDIAN_FLOW_Parameters, IO) FileStorage fsReader(serializedParameters, FileStorage::READ + FileStorage::MEMORY); - TrackerMedianFlow::Params readParameters; + legacy::TrackerMedianFlow::Params readParameters; readParameters.read(fsReader.root()); ASSERT_EQ(parameters.maxLevel, readParameters.maxLevel); @@ -41,11 +45,11 @@ TEST(MEDIAN_FLOW_Parameters, IO) TEST(MEDIAN_FLOW_Parameters, Default_Value_If_Absent) { - TrackerMedianFlow::Params defaultParameters; + legacy::TrackerMedianFlow::Params defaultParameters; FileStorage fsReader(String("%YAML 1.0"), FileStorage::READ + FileStorage::MEMORY); - TrackerMedianFlow::Params readParameters; + legacy::TrackerMedianFlow::Params readParameters; readParameters.read(fsReader.root()); ASSERT_EQ(defaultParameters.maxLevel, readParameters.maxLevel); @@ -60,7 +64,7 @@ TEST(MEDIAN_FLOW_Parameters, Default_Value_If_Absent) TEST(KCF_Parameters, IO) { - TrackerKCF::Params parameters; + legacy::TrackerKCF::Params parameters; parameters.sigma = 0.3f; parameters.lambda = 0.02f; @@ -83,7 +87,7 @@ TEST(KCF_Parameters, IO) FileStorage fsReader(serializedParameters, FileStorage::READ + FileStorage::MEMORY); - TrackerKCF::Params readParameters; + legacy::TrackerKCF::Params readParameters; readParameters.read(fsReader.root()); ASSERT_DOUBLE_EQ(parameters.sigma, readParameters.sigma); @@ -103,11 +107,11 @@ TEST(KCF_Parameters, IO) TEST(KCF_Parameters, Default_Value_If_Absent) { - TrackerKCF::Params defaultParameters; + legacy::TrackerKCF::Params defaultParameters; FileStorage fsReader(String("%YAML 1.0"), FileStorage::READ + FileStorage::MEMORY); - TrackerKCF::Params readParameters; + legacy::TrackerKCF::Params readParameters; readParameters.read(fsReader.root()); ASSERT_DOUBLE_EQ(defaultParameters.sigma, readParameters.sigma); diff --git a/modules/tracking/test/test_trackers.cpp b/modules/tracking/test/test_trackers.cpp index db1617e35..222d272ca 100644 --- a/modules/tracking/test/test_trackers.cpp +++ b/modules/tracking/test/test_trackers.cpp @@ -41,7 +41,16 @@ #include "test_precomp.hpp" +#define TEST_LEGACY +#include + +//#define DEBUG_TEST +#ifdef DEBUG_TEST +#include +#endif + namespace opencv_test { namespace { +//using namespace cv::tracking; #define TESTSET_NAMES testing::Values("david","dudek","faceocc2") @@ -73,19 +82,52 @@ enum BBTransformations Scale_1_2 = 12 }; +namespace { + +std::vector splitString(const std::string& s_, const std::string& delimiter) +{ + std::string s = s_; + std::vector token; + size_t pos = 0; + while ( ( pos = s.find( delimiter ) ) != std::string::npos ) + { + token.push_back( s.substr( 0, pos ) ); + s.erase( 0, pos + delimiter.length() ); + } + token.push_back( s ); + return token; +} + +float calcDistance(const Rect& a, const Rect& b) +{ + Point2f p_a( (float)(a.x + a.width / 2), (float)(a.y + a.height / 2) ); + Point2f p_b( (float)(b.x + b.width / 2), (float)(b.y + b.height / 2) ); + return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); +} + +float calcOverlap(const Rect& a, const Rect& b) +{ + float rectIntersectionArea = (float)(a & b).area(); + return rectIntersectionArea / (a.area() + b.area() - rectIntersectionArea); +} + +} // namespace + + +template class TrackerTest { - public: +public: - TrackerTest(Ptr _tracker, string _video, float _distanceThreshold, - float _overlapThreshold, int _shift = NoTransform, int _segmentIdx = 1, int _numSegments = 10 ); - virtual ~TrackerTest(); - virtual void run(); + TrackerTest(const Ptr& tracker, const string& video, float distanceThreshold, + float overlapThreshold, int shift = NoTransform, int segmentIdx = 1, int numSegments = 10); + ~TrackerTest() {} + void run(); - protected: +protected: void checkDataTest(); - void distanceAndOvrerlapTest(); + void distanceAndOverlapTest(); Ptr tracker; string video; @@ -103,16 +145,13 @@ class TrackerTest int endFrame; vector validSequence; - private: - float calcDistance( Rect a, Rect b ); - float calcOverlap( Rect a, Rect b ); - Rect applyShift(Rect bb); - std::vector splitString( std::string s, std::string delimiter ); - +private: + Rect applyShift(const Rect& bb); }; -TrackerTest::TrackerTest(Ptr _tracker, string _video, float _distanceThreshold, - float _overlapThreshold, int _shift, int _segmentIdx, int _numSegments ) : +template +TrackerTest::TrackerTest(const Ptr& _tracker, const string& _video, float _distanceThreshold, + float _overlapThreshold, int _shift, int _segmentIdx, int _numSegments ) : tracker( _tracker ), video( _video ), overlapThreshold( _overlapThreshold ), @@ -121,41 +160,13 @@ TrackerTest::TrackerTest(Ptr _tracker, string _video, float _distanceTh shift(_shift), numSegments(_numSegments) { + // nothing } -TrackerTest::~TrackerTest() -{ - -} - -std::vector TrackerTest::splitString( std::string s, std::string delimiter ) -{ - std::vector token; - size_t pos = 0; - while ( ( pos = s.find( delimiter ) ) != std::string::npos ) - { - token.push_back( s.substr( 0, pos ) ); - s.erase( 0, pos + delimiter.length() ); - } - token.push_back( s ); - return token; -} - -float TrackerTest::calcDistance( Rect a, Rect b ) -{ - Point2f p_a( (float)(a.x + a.width / 2), (float)(a.y + a.height / 2) ); - Point2f p_b( (float)(b.x + b.width / 2), (float)(b.y + b.height / 2) ); - return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); -} - -float TrackerTest::calcOverlap( Rect a, Rect b ) -{ - float rectIntersectionArea = (float)(a & b).area(); - return rectIntersectionArea / (a.area() + b.area() - rectIntersectionArea); -} - -Rect TrackerTest::applyShift(Rect bb) +template +Rect TrackerTest::applyShift(const Rect& bb_) { + Rect bb = bb_; Point center( bb.x + ( bb.width / 2 ), bb.y + ( bb.height / 2 ) ); int xLimit = bb.x + bb.width - 1; @@ -244,51 +255,77 @@ Rect TrackerTest::applyShift(Rect bb) return bb; } -void TrackerTest::distanceAndOvrerlapTest() +template +void TrackerTest::distanceAndOverlapTest() { - Mat frame; bool initialized = false; int fc = ( startFrame - gtStartFrame ); bbs.at( fc ) = applyShift(bbs.at( fc )); Rect currentBBi = bbs.at( fc ); - Rect2d currentBB(currentBBi); + ROI_t currentBB(currentBBi); float sumDistance = 0; float sumOverlap = 0; string folder = cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + string videoPath = folder + "/" + video + ".webm"; VideoCapture c; - c.open( folder + "/" + video + ".webm" ); - c.set( CAP_PROP_POS_FRAMES, startFrame ); + c.open(videoPath); + ASSERT_TRUE(c.isOpened()) << videoPath; +#if 0 + c.set(CAP_PROP_POS_FRAMES, startFrame); +#else + if (startFrame) + std::cout << "startFrame = " << startFrame << std::endl; + for (int i = 0; i < startFrame; i++) + { + Mat dummy_frame; + c >> dummy_frame; + ASSERT_FALSE(dummy_frame.empty()) << i << ": " << videoPath; + } +#endif for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) { + Mat frame; c >> frame; - if( frame.empty() ) - { - break; - } + ASSERT_FALSE(frame.empty()) << "frameCounter=" << frameCounter << " video=" << videoPath; if( !initialized ) { +#if 0 if( !tracker->init( frame, currentBB ) ) { FAIL()<< "Could not initialize tracker" << endl; return; } +#else + tracker->init(frame, currentBB); +#endif + std::cout << "frame size = " << frame.size() << std::endl; initialized = true; } else if( initialized ) { if( frameCounter >= (int) bbs.size() ) - break; + break; tracker->update( frame, currentBB ); } float curDistance = calcDistance( currentBB, bbs.at( fc ) ); float curOverlap = calcOverlap( currentBB, bbs.at( fc ) ); +#ifdef DEBUG_TEST + Mat result; + repeat(frame, 1, 2, result); + rectangle(result, currentBB, Scalar(0, 255, 0), 1); + Rect roi2(frame.cols, 0, frame.cols, frame.rows); + rectangle(result(roi2), bbs.at(fc), Scalar(0, 0, 255), 1); + imshow("result", result); + waitKey(1); +#endif + sumDistance += curDistance; sumOverlap += curOverlap; fc++; @@ -297,20 +334,12 @@ void TrackerTest::distanceAndOvrerlapTest() float meanDistance = sumDistance / (endFrame - startFrame); float meanOverlap = sumOverlap / (endFrame - startFrame); - if( meanDistance > distanceThreshold ) - { - FAIL()<< "Incorrect distance: curr = " << meanDistance << ", max = " << distanceThreshold << endl; - return; - } - - if( meanOverlap < overlapThreshold ) - { - FAIL()<< "Incorrect overlap: curr = " << meanOverlap << ", min = " << overlapThreshold << endl; - return; - } + EXPECT_LE(meanDistance, distanceThreshold); + EXPECT_GE(meanOverlap, overlapThreshold); } -void TrackerTest::checkDataTest() +template +void TrackerTest::checkDataTest() { FileStorage fs; @@ -324,10 +353,7 @@ void TrackerTest::checkDataTest() std::ifstream gt; //open the ground truth gt.open( gtFile.c_str() ); - if( !gt.is_open() ) - { - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - } + ASSERT_TRUE(gt.is_open()) << gtFile; string line; int bbCounter = 0; while ( getline( gt, line ) ) @@ -372,20 +398,14 @@ void TrackerTest::checkDataTest() std::ifstream gt2; //open the ground truth gt2.open( gtFile.c_str() ); - if( !gt2.is_open() ) - { - FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; - } + ASSERT_TRUE(gt2.is_open()) << gtFile; string line2; int bbCounter2 = 0; while ( getline( gt2, line2 ) ) { vector tokens = splitString( line2, "," ); Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ); - if( tokens.size() != 4 ) - { - FAIL()<< "Incorrect ground truth file"; - } + ASSERT_EQ((size_t)4, tokens.size()) << "Incorrect ground truth file " << gtFile; bbs.push_back( bb ); bbCounter2++; @@ -396,17 +416,12 @@ void TrackerTest::checkDataTest() endFrame = (int)bbs.size(); } -void TrackerTest::run() +template +void TrackerTest::run() { - srand( 1 ); - - SCOPED_TRACE( "A" ); + srand( 1 ); // FIXIT remove that, ensure that there is no "rand()" in implementation - if( !tracker ) - { - FAIL()<< "Error in the instantiation of the tracker" << endl; - return; - } + ASSERT_TRUE(tracker); checkDataTest(); @@ -414,7 +429,7 @@ void TrackerTest::run() if( ::testing::Test::HasFatalFailure() ) return; - distanceAndOvrerlapTest(); + distanceAndOverlapTest(); } /****************************************************************************************\ @@ -433,167 +448,240 @@ PARAM_TEST_CASE(DistanceAndOverlap, string) TEST_P(DistanceAndOverlap, MedianFlow) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 35, .5f, NoTransform, 1, 1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 35, .5f, NoTransform, 1, 1); test.run(); } TEST_P(DistanceAndOverlap, MIL) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .65f, NoTransform); + TrackerTest test(TrackerMIL::create(), dataset, 30, .65f, NoTransform); + test.run(); +} +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, MIL_legacy) +{ + TrackerTest test(legacy::TrackerMIL::create(), dataset, 30, .65f, NoTransform); test.run(); } +#endif TEST_P(DistanceAndOverlap, Boosting) { - TrackerTest test( TrackerBoosting::create(), dataset, 70, .7f, NoTransform); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 70, .7f, NoTransform); test.run(); } TEST_P(DistanceAndOverlap, KCF) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); + TrackerTest test(TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); test.run(); } +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, KCF_legacy) +{ + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .35f, NoTransform, 5); + test.run(); +} +#endif TEST_P(DistanceAndOverlap, TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 40, .45f, NoTransform); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 40, .45f, NoTransform); test.run(); } TEST_P(DistanceAndOverlap, MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 22, .7f, NoTransform); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 22, .7f, NoTransform); test.run(); } TEST_P(DistanceAndOverlap, CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 22, .7f, NoTransform); + TrackerTest test(TrackerCSRT::create(), dataset, 22, .7f, NoTransform); test.run(); } +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, CSRT_legacy) +{ + TrackerTest test(legacy::TrackerCSRT::create(), dataset, 22, .7f, NoTransform); + test.run(); +} +#endif /***************************************************************************************/ //Tests with shifted initial window TEST_P(DistanceAndOverlap, Shifted_Data_MedianFlow) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 80, .2f, CenterShiftLeft, 1, 1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 80, .2f, CenterShiftLeft, 1, 1); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_MIL) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .6f, CenterShiftLeft); + TrackerTest test(TrackerMIL::create(), dataset, 30, .6f, CenterShiftLeft); test.run(); } +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Shifted_Data_MIL_legacy) +{ + TrackerTest test(legacy::TrackerMIL::create(), dataset, 30, .6f, CenterShiftLeft); + test.run(); +} +#endif TEST_P(DistanceAndOverlap, Shifted_Data_Boosting) { - TrackerTest test( TrackerBoosting::create(), dataset, 80, .65f, CenterShiftLeft); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 80, .65f, CenterShiftLeft); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_KCF) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); + TrackerTest test(TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); + test.run(); +} +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Shifted_Data_KCF_legacy) +{ + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .4f, CenterShiftLeft, 5); test.run(); } +#endif TEST_P(DistanceAndOverlap, Shifted_Data_TLD) { - TrackerTest test( TrackerTLD::create(), dataset, 30, .35f, CenterShiftLeft); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 30, .35f, CenterShiftLeft); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 13, .69f, CenterShiftLeft); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 13, .69f, CenterShiftLeft); test.run(); } TEST_P(DistanceAndOverlap, Shifted_Data_CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); + TrackerTest test(TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); + test.run(); +} +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Shifted_Data_CSRT_legacy) +{ + TrackerTest test(legacy::TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft); test.run(); } +#endif + /***************************************************************************************/ //Tests with scaled initial window TEST_P(DistanceAndOverlap, Scaled_Data_MedianFlow) { - TrackerTest test( TrackerMedianFlow::create(), dataset, 25, .5f, Scale_1_1, 1, 1); + TrackerTest test(legacy::TrackerMedianFlow::create(), dataset, 25, .5f, Scale_1_1, 1, 1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_MIL) { - TrackerTest test( TrackerMIL::create(), dataset, 30, .7f, Scale_1_1); + TrackerTest test(TrackerMIL::create(), dataset, 30, .7f, Scale_1_1); + test.run(); +} +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Scaled_Data_MIL_legacy) +{ + TrackerTest test(legacy::TrackerMIL::create(), dataset, 30, .7f, Scale_1_1); test.run(); } +#endif TEST_P(DistanceAndOverlap, Scaled_Data_Boosting) { - TrackerTest test( TrackerBoosting::create(), dataset, 80, .7f, Scale_1_1); + TrackerTest test(legacy::TrackerBoosting::create(), dataset, 80, .7f, Scale_1_1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_KCF) { - TrackerTest test( TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); + TrackerTest test(TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); test.run(); } - -TEST_P(DistanceAndOverlap, Scaled_Data_TLD) +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Scaled_Data_KCF_legacy) { - TrackerTest test( TrackerTLD::create(), dataset, 30, .45f, Scale_1_1); + TrackerTest test(legacy::TrackerKCF::create(), dataset, 20, .4f, Scale_1_1, 5); test.run(); } +#endif - -TEST_P(DistanceAndOverlap, DISABLED_GOTURN) +TEST_P(DistanceAndOverlap, Scaled_Data_TLD) { - TrackerTest test(TrackerGOTURN::create(), dataset, 18, .5f, NoTransform); + TrackerTest test(legacy::TrackerTLD::create(), dataset, 30, .45f, Scale_1_1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_MOSSE) { - TrackerTest test( TrackerMOSSE::create(), dataset, 22, 0.69f, Scale_1_1, 1); + TrackerTest test(legacy::TrackerMOSSE::create(), dataset, 22, 0.69f, Scale_1_1, 1); test.run(); } TEST_P(DistanceAndOverlap, Scaled_Data_CSRT) { - TrackerTest test( TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); + TrackerTest test(TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); + test.run(); +} +#ifdef TEST_LEGACY +TEST_P(DistanceAndOverlap, Scaled_Data_CSRT_legacy) +{ + TrackerTest test(TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1); test.run(); } +#endif + +TEST_P(DistanceAndOverlap, GOTURN) +{ + std::string model = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.prototxt"); + std::string weights = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.caffemodel", false); + cv::TrackerGOTURN::Params params; + params.modelTxt = model; + params.modelBin = weights; + TrackerTest test(TrackerGOTURN::create(params), dataset, 35, .35f, NoTransform); + test.run(); +} + +INSTANTIATE_TEST_CASE_P(Tracking, DistanceAndOverlap, TESTSET_NAMES); + + TEST(GOTURN, memory_usage) { - cv::Rect2d roi(145, 70, 85, 85); - cv::Mat frame; + cv::Rect roi(145, 70, 85, 85); std::string model = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.prototxt"); std::string weights = cvtest::findDataFile("dnn/gsoc2016-goturn/goturn.caffemodel", false); cv::TrackerGOTURN::Params params; params.modelTxt = model; params.modelBin = weights; - cv::Ptr tracker = cv::TrackerGOTURN::create(params); + cv::Ptr tracker = TrackerGOTURN::create(params); + string inputVideo = cvtest::findDataFile("tracking/david/data/david.webm"); cv::VideoCapture video(inputVideo); + ASSERT_TRUE(video.isOpened()) << inputVideo; + cv::Mat frame; video >> frame; + ASSERT_FALSE(frame.empty()) << inputVideo; tracker->init(frame, roi); string ground_truth_bb; for (int nframes = 0; nframes < 15; ++nframes) { std::cout << "Frame: " << nframes << std::endl; video >> frame; - tracker->update(frame, roi); + bool res = tracker->update(frame, roi); + ASSERT_TRUE(res); std::cout << "Predicted ROI: " << roi << std::endl; } } -INSTANTIATE_TEST_CASE_P( Tracking, DistanceAndOverlap, TESTSET_NAMES); - }} // namespace -/* End of file. */ diff --git a/modules/tracking/test/test_ukf.cpp b/modules/tracking/test/test_ukf.cpp index 60196ee1d..c8ebe3edc 100644 --- a/modules/tracking/test/test_ukf.cpp +++ b/modules/tracking/test/test_ukf.cpp @@ -43,7 +43,7 @@ #include "opencv2/tracking/kalman_filters.hpp" namespace opencv_test { namespace { -using namespace cv::tracking; +using namespace cv::detail; // In this two tests Unscented Kalman Filter are applied to the dynamic system from example "The reentry problem" from // "A New Extension of the Kalman Filter to Nonlinear Systems" by Simon J. Julier and Jeffrey K. Uhlmann. diff --git a/modules/tracking/tutorials/tutorial_multitracker.markdown b/modules/tracking/tutorials/tutorial_multitracker.markdown index f9c7605a5..0dce97fc4 100644 --- a/modules/tracking/tutorials/tutorial_multitracker.markdown +++ b/modules/tracking/tutorials/tutorial_multitracker.markdown @@ -38,11 +38,11 @@ Explanation You can add all tracked objects at once to the MultiTracker as shown in the code. In this case, all objects will be tracked using same tracking algorithm as specified in decaration of MultiTracker object. If you want to use different tracker algorithms for each tracked object, - You should add the tracked objects one by one and specify their tracking algorithm using the variant of @ref cv::MultiTracker::add. - @sa cv::MultiTracker::add( const String& trackerType, const Mat& image, const Rect2d& boundingBox ) + You should add the tracked objects one by one and specify their tracking algorithm using the variant of @ref cv::legacy::MultiTracker::add. + @sa cv::legacy::MultiTracker::add( const String& trackerType, const Mat& image, const Rect2d& boundingBox ) -# **Obtaining the result** @snippet tracking/samples/tutorial_multitracker.cpp result - You can access the result from the public variable @ref cv::MultiTracker::objects provided by the MultiTracker class as shown in the code. + You can access the result from the public variable @ref cv::legacy::MultiTracker::objects provided by the MultiTracker class as shown in the code.