diff --git a/modules/tracking/perf/perf_Tracker.cpp b/modules/tracking/perf/perf_Tracker.cpp new file mode 100644 index 000000000..5bd5a9ac6 --- /dev/null +++ b/modules/tracking/perf/perf_Tracker.cpp @@ -0,0 +1,273 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "perf_precomp.hpp" +#include + +using namespace std; +using namespace cv; +using namespace perf; +using std::tr1::make_tuple; +using std::tr1::get; + +//write sanity: ./bin/opencv_perf_tracking --perf_write_sanity=true --perf_min_samples=1 +//verify sanity: ./bin/opencv_perf_tracking --perf_min_samples=1 + +#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") +//#define TESTSET_NAMES testing::internal::ValueArray1("david") +#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + +const string TRACKING_DIR = "cv/tracking"; +const string FOLDER_IMG = "data"; + +typedef perf::TestBaseWithParam > tracking; + +std::vector splitString( std::string s, std::string delimiter ) +{ + std::vector token; + size_t pos = 0; + while ( ( pos = s.find( delimiter ) ) != std::string::npos ) + { + token.push_back( s.substr( 0, pos ) ); + s.erase( 0, pos + delimiter.length() ); + } + token.push_back( s ); + return token; +} + +void checkData( const string& datasetMeta, int& startFrame, string& prefix, string& suffix ) +{ + //get informations on the current test data + FileStorage fs; + fs.open( datasetMeta, FileStorage::READ ); + fs["start"] >> startFrame; + fs["prefix"] >> prefix; + fs["suffix"] >> suffix; + fs.release(); +} + +bool getGroundTruth( const string& gtFile, vector& gtBBs ) +{ + ifstream gt; + //open the ground truth + gt.open( gtFile.c_str() ); + if( !gt.is_open() ) + { + return false; + } + string line; + Rect currentBB; + while ( getline( gt, line ) ) + { + vector tokens = splitString( line, "," ); + + if( tokens.size() != 4 ) + { + return false; + } + + gtBBs.push_back( + Rect( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ) ); + } + return true; +} + +void getSegment( int segmentId, int numSegments, int bbCounter, int& startFrame, int& endFrame ) +{ + //compute the start and the and for each segment + int gtStartFrame = startFrame; + int numFrame = bbCounter / numSegments; + startFrame += ( segmentId - 1 ) * numFrame; + endFrame = startFrame + numFrame; + + if( ( segmentId ) == numSegments ) + endFrame = bbCounter + gtStartFrame - 1; +} + +void getMatOfRects( const vector& bbs, Mat& bbs_mat ) +{ + for ( size_t b = 0; b < bbs.size(); b++ ) + { + bbs_mat.at( b, 0 ) = bbs[b].x; + bbs_mat.at( b, 1 ) = bbs[b].y; + bbs_mat.at( b, 2 ) = bbs[b].width; + bbs_mat.at( b, 3 ) = bbs[b].height; + } +} + +PERF_TEST_P(tracking, mil, testing::Combine(TESTSET_NAMES, SEGMENTS)) +{ + string video = get<0>( GetParam() ); + int segmentId = get<1>( GetParam() ); + + int startFrame; + string prefix; + string suffix; + string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); + checkData( datasetMeta, startFrame, prefix, suffix ); + int gtStartFrame = startFrame; + + vector gtBBs; + string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); + if( !getGroundTruth( gtFile, gtBBs ) ) + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + int bbCounter = gtBBs.size(); + + Mat frame; + bool initialized = false; + vector bbs; + + Ptr tracker = Tracker::create( "MIL" ); + string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); + int endFrame = 0; + getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); + + Rect currentBB = gtBBs[startFrame - gtStartFrame]; + + TEST_CYCLE_N(1) + { + VideoCapture c; + c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) + { + c >> frame; + + if( frame.empty() ) + { + break; + } + + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + tracker->update( frame, currentBB ); + } + bbs.push_back( currentBB ); + + } + } + //save the bounding boxes in a Mat + Mat bbs_mat( bbs.size(), 4, CV_32F ); + getMatOfRects( bbs, bbs_mat ); + + SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); + +} + +PERF_TEST_P(tracking, boosting, testing::Combine(TESTSET_NAMES, SEGMENTS)) +{ + string video = get<0>( GetParam() ); + int segmentId = get<1>( GetParam() ); + + int startFrame; + string prefix; + string suffix; + string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" ); + checkData( datasetMeta, startFrame, prefix, suffix ); + int gtStartFrame = startFrame; + + vector gtBBs; + string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" ); + if( !getGroundTruth( gtFile, gtBBs ) ) + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + int bbCounter = gtBBs.size(); + + Mat frame; + bool initialized = false; + vector bbs; + + Ptr tracker = Tracker::create( "BOOSTING" ); + string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) ); + int endFrame = 0; + getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame ); + + Rect currentBB = gtBBs[startFrame - gtStartFrame]; + + TEST_CYCLE_N(1) + { + VideoCapture c; + c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) + { + c >> frame; + + if( frame.empty() ) + { + break; + } + + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + tracker->update( frame, currentBB ); + } + bbs.push_back( currentBB ); + + } + } + //save the bounding boxes in a Mat + Mat bbs_mat( bbs.size(), 4, CV_32F ); + getMatOfRects( bbs, bbs_mat ); + + SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE ); + +} diff --git a/modules/tracking/perf/perf_tracking.cpp b/modules/tracking/perf/perf_tracking.cpp deleted file mode 100644 index c143a62ce..000000000 --- a/modules/tracking/perf/perf_tracking.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "perf_precomp.hpp" - -using namespace std; -using namespace cv; -using namespace perf; diff --git a/modules/tracking/samples/tracker.cpp b/modules/tracking/samples/tracker.cpp index 80ef98f90..32c3cac5a 100644 --- a/modules/tracking/samples/tracker.cpp +++ b/modules/tracking/samples/tracker.cpp @@ -13,15 +13,17 @@ static bool selectObject = false; static bool startSelection = false; static const char* keys = -{ "{@tracker_algorithm | | tracker algorithm }" - "{@video_name | | video name }" }; +{ "{@tracker_algorithm | | Tracker algorithm }" + "{@video_name | | video name }" + "{@start_frame |1| Start frame }" }; static void help() { cout << "\nThis example shows the functionality of \"Long-term optical tracking API\"" "-- pause video [p] and draw a bounding box around the target to start the tracker\n" + "Example of is in opencv_extra/testdata/cv/tracking/\n" "Call:\n" - "./tracker \n" + "./tracker \n" << endl; cout << "\n\nHot keys: \n" @@ -69,6 +71,7 @@ int main( int argc, char** argv ) String tracker_algorithm = parser.get( 0 ); String video_name = parser.get( 1 ); + int start_frame = parser.get( 2 ); if( tracker_algorithm.empty() || video_name.empty() ) { @@ -79,6 +82,7 @@ int main( int argc, char** argv ) //open the capture VideoCapture cap; cap.open( video_name ); + cap.set( CAP_PROP_POS_FRAMES, start_frame ); if( !cap.isOpened() ) { @@ -108,11 +112,19 @@ int main( int argc, char** argv ) imshow( "Tracking API", image ); bool initialized = false; + int frameCounter = 0; + for ( ;; ) { if( !paused ) { cap >> frame; + + if( frame.empty() ) + { + break; + } + frame.copyTo( image ); if( !initialized && selectObject ) @@ -134,6 +146,7 @@ int main( int argc, char** argv ) } } imshow( "Tracking API", image ); + frameCounter++; } char c = (char) waitKey( 2 ); diff --git a/modules/tracking/src/trackerBoosting.cpp b/modules/tracking/src/trackerBoosting.cpp index 5a158d0ee..123aee8a5 100644 --- a/modules/tracking/src/trackerBoosting.cpp +++ b/modules/tracking/src/trackerBoosting.cpp @@ -56,7 +56,7 @@ TrackerBoosting::Params::Params() { numClassifiers = 100; samplerOverlap = 0.99f; - samplerSearchFactor = 2; + samplerSearchFactor = 1.8; iterationInit = 50; featureSetNumFeatures = ( numClassifiers * 10 ) + iterationInit; } @@ -108,6 +108,7 @@ void TrackerBoosting::write( cv::FileStorage& fs ) const bool TrackerBoosting::initImpl( const Mat& image, const Rect& boundingBox ) { + srand (1); //sampling Mat_ intImage; Mat_ intSqImage; diff --git a/modules/tracking/src/trackerBoostingModel.hpp b/modules/tracking/src/trackerBoostingModel.hpp index 6fa69b032..1e5e319f0 100644 --- a/modules/tracking/src/trackerBoostingModel.hpp +++ b/modules/tracking/src/trackerBoostingModel.hpp @@ -99,7 +99,6 @@ class TrackerBoostingModel : public TrackerModel private: std::vector currentSample; - std::vector > meanSigmaPair; int mode; }; diff --git a/modules/tracking/src/trackerMIL.cpp b/modules/tracking/src/trackerMIL.cpp index a20f9939c..7878041f3 100644 --- a/modules/tracking/src/trackerMIL.cpp +++ b/modules/tracking/src/trackerMIL.cpp @@ -124,6 +124,7 @@ void TrackerMIL::compute_integral( const Mat & img, Mat & ii_img ) bool TrackerMIL::initImpl( const Mat& image, const Rect& boundingBox ) { + srand (1); Mat intImage; compute_integral( image, intImage ); TrackerSamplerCSC::Params CSCparameters; diff --git a/modules/tracking/test/test_tracker.cpp b/modules/tracking/test/test_tracker.cpp deleted file mode 100644 index 900772b92..000000000 --- a/modules/tracking/test/test_tracker.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // - // Copyright (C) 2013, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ - -#include "test_precomp.hpp" -#include "opencv2/tracking.hpp" - -using namespace cv; -using namespace std; - -class CV_TrackerBaseTest : public cvtest::BaseTest -{ - public: - CV_TrackerBaseTest(); - virtual ~CV_TrackerBaseTest(); - -}; - -CV_TrackerBaseTest::CV_TrackerBaseTest() -{ - -} - -CV_TrackerBaseTest::~CV_TrackerBaseTest() -{ - -} - -/************************************ TrackerMIL ************************************/ - -class CV_TrackerMILTest : public CV_TrackerBaseTest -{ - public: - CV_TrackerMILTest(); - ~CV_TrackerMILTest(); - - protected: - void run( int ); -}; - -CV_TrackerMILTest::CV_TrackerMILTest() -{ -} - -CV_TrackerMILTest::~CV_TrackerMILTest() -{ -} - -void CV_TrackerMILTest::run( int ) -{ - ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC ); - ts->printf( cvtest::TS::LOG, "CV_TrackerMILTest to be implemented" ); -} - -TEST(DISABLED_Tracking_TrackerMIL, accuracy) -{ - CV_TrackerMILTest test; - test.safe_run(); -} - -/************************************ TrackerBoosting ************************************/ - -class CV_TrackerBoostingTest : public CV_TrackerBaseTest -{ - public: - CV_TrackerBoostingTest(); - ~CV_TrackerBoostingTest(); - - protected: - void run( int ); -}; - -CV_TrackerBoostingTest::CV_TrackerBoostingTest() -{ -} - -CV_TrackerBoostingTest::~CV_TrackerBoostingTest() -{ -} - -void CV_TrackerBoostingTest::run( int ) -{ - ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC ); - ts->printf( cvtest::TS::LOG, "CV_TrackerBoostingTest to be implemented" ); -} - -TEST(DISABLED_Tracking_TrackerBoosting, accuracy) -{ - CV_TrackerBoostingTest test; - test.safe_run(); -} - -/* End of file. */ diff --git a/modules/tracking/test/test_trackerOPE.cpp b/modules/tracking/test/test_trackerOPE.cpp new file mode 100644 index 000000000..ba45dff90 --- /dev/null +++ b/modules/tracking/test/test_trackerOPE.cpp @@ -0,0 +1,413 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "test_precomp.hpp" +#include "opencv2/tracking.hpp" +#include + +using namespace cv; +using namespace testing; +using namespace std; + +#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > > +#define GET_PARAM(k) std::tr1::get< k >(GetParam()) +#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") +#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50) +#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1) + +const string TRACKING_DIR = "tracking"; +const string FOLDER_IMG = "data"; + +/* + * The Evaluation Methodologies are partially based on: + * ==================================================================================================================== + * [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013 + * + */ + +//Robustness Evaluation, see [OTB] chapter 4. OPE One Pass Evaluation +class TrackerOPETest +{ + public: + enum + { + DISTANCE = 1, // test trackers based on euclidean distance + OVERLAP = 2 // test trackers based on the overlapping ratio + }; + + TrackerOPETest( Ptr _tracker, int _testType, string _video, float _threshold ); + virtual ~TrackerOPETest(); + virtual void run(); + string getRatioSucc() const; + + protected: + void checkDataTest(); + + void distanceTest(); + void overlapTest(); + + Ptr tracker; + int testType; + string video; + std::vector bbs; + int startFrame; + string suffix; + string prefix; + float threshold; + float ratioSucc; + + private: + float calcDistance( Rect a, Rect b ); + float calcOverlap( Rect a, Rect b ); + std::vector splitString( std::string s, std::string delimiter ); + +}; + +TrackerOPETest::TrackerOPETest( Ptr _tracker, int _testType, string _video, float _threshold ) : + tracker( _tracker ), + testType( _testType ), + video( _video ), + threshold( _threshold ) +{ + startFrame = 1; + ratioSucc = 0; +} + +TrackerOPETest::~TrackerOPETest() +{ + +} + +string TrackerOPETest::getRatioSucc() const +{ + stringstream ratio; + ratio << ratioSucc; + return ratio.str(); +} + +std::vector TrackerOPETest::splitString( std::string s, std::string delimiter ) +{ + std::vector token; + size_t pos = 0; + while ( ( pos = s.find( delimiter ) ) != std::string::npos ) + { + token.push_back( s.substr( 0, pos ) ); + s.erase( 0, pos + delimiter.length() ); + } + token.push_back( s ); + return token; +} + +float TrackerOPETest::calcDistance( Rect a, Rect b ) +{ + Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 ); + Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 ); + return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); +} + +float TrackerOPETest::calcOverlap( Rect a, Rect b ) +{ + float aArea = a.width * a.height; + float bArea = b.width * b.height; + + if( aArea < bArea ) + { + a.x -= ( b.width - a.width ) / 2; + a.y -= ( b.height - a.height ) / 2; + a.width = b.width; + a.height = b.height; + } + else + { + b.x -= ( a.width - b.width ) / 2; + b.y -= ( a.height - b.height ) / 2; + b.width = a.width; + b.height = a.height; + } + + Rect rectIntersection = a & b; + Rect rectUnion = a | b; + float iArea = rectIntersection.width * rectIntersection.height; + float uArea = rectUnion.width * rectUnion.height; + float overlap = iArea / uArea; + return overlap; +} + +void TrackerOPETest::distanceTest() +{ + Mat frame; + bool initialized = false; + + Rect currentBB = bbs.at( 0 ); + float sumDistance = 0; + int frameCounter = 0; + int frameCounterSucc = 0; + string folder = cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + for ( ;; ) + { + c >> frame; + + if( frame.empty() ) + { + break; + } + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + float curDistance = calcDistance( currentBB, bbs.at( frameCounter ) ); + if( curDistance <= threshold ) + frameCounterSucc++; + sumDistance += curDistance; + frameCounter++; + } + + float distance = sumDistance / frameCounter; + ratioSucc = (float) frameCounterSucc / (float) frameCounter; + + if( distance > threshold ) + { + FAIL()<< "Incorrect distance: curr = " << distance << ", max = " << threshold << endl; + return; + } + +} + +void TrackerOPETest::overlapTest() +{ + Mat frame; + bool initialized = false; + Rect currentBB = bbs.at( 0 ); + float sumOverlap = 0; + string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + int frameCounter = 0; + int frameCounterSucc = 0; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( ;; ) + { + c >> frame; + if( frame.empty() ) + { + break; + } + + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + float curOverlap = calcOverlap( currentBB, bbs.at( frameCounter ) ); + if( curOverlap >= threshold ) + frameCounterSucc++; + + sumOverlap += curOverlap; + frameCounter++; + } + + float overlap = sumOverlap / frameCounter; + ratioSucc = (float) frameCounterSucc / (float) frameCounter; + + if( overlap < threshold ) + { + FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl; + return; + } +} + +void TrackerOPETest::checkDataTest() +{ + string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt"; + + ifstream gt; + //open the ground truth + gt.open( gtFile.c_str() ); + if( !gt.is_open() ) + { + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + } + string line; + int bbCounter = 0; + while ( getline( gt, line ) ) + { + vector tokens = splitString( line, "," ); + Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ); + if( tokens.size() != 4 ) + { + FAIL()<< "Incorrect ground truth file"; + } + bbs.push_back( bb ); + bbCounter++; + } + + FileStorage fs; + fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ ); + fs["start"] >> startFrame; + fs["prefix"] >> prefix; + fs["suffix"] >> suffix; + fs.release(); + +} + +void TrackerOPETest::run() +{ + srand( 1 ); + + SCOPED_TRACE( "A" ); + + if( !tracker ) + { + FAIL()<< "Error in the instantiation of the tracker" << endl; + return; + } + + checkDataTest(); + + //check for failure + if( ::testing::Test::HasFatalFailure() ) + return; + + if( testType == DISTANCE ) + { + distanceTest(); + } + else if( testType == OVERLAP ) + { + overlapTest(); + } + else + { + FAIL()<< "Test type unknown" << endl; + return; + } + +} + +/****************************************************************************************\ +* Tests registrations * + \****************************************************************************************/ + +//[TESTDATA] [LOCATION ERROR THRESHOLD] +PARAM_TEST_CASE(OPE_Distance, string, float) +{ + string dataset; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + threshold = GET_PARAM(1); + } +}; + +//[TESTDATA] [OVERLAP THRESHOLD] +PARAM_TEST_CASE(OPE_Overlap, string, float) +{ + string dataset; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + threshold = GET_PARAM(1); + } +}; + +TEST_P(OPE_Distance, MIL) +{ + TrackerOPETest test( Tracker::create( "MIL" ), TrackerOPETest::DISTANCE, dataset, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(OPE_Overlap, MIL) +{ + TrackerOPETest test( Tracker::create( "MIL" ), TrackerOPETest::OVERLAP, dataset, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(OPE_Distance, Boosting) +{ + TrackerOPETest test( Tracker::create( "BOOSTING" ), TrackerOPETest::DISTANCE, dataset, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(OPE_Overlap, Boosting) +{ + TrackerOPETest test( Tracker::create( "BOOSTING" ), TrackerOPETest::OVERLAP, dataset, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +INSTANTIATE_TEST_CASE_P( Tracking, OPE_Distance, testing::Combine( TESTSET_NAMES, LOCATION_ERROR_THRESHOLD ) ); + +INSTANTIATE_TEST_CASE_P( Tracking, OPE_Overlap, testing::Combine( TESTSET_NAMES, OVERLAP_THRESHOLD ) ); + +/* End of file. */ diff --git a/modules/tracking/test/test_trackerSRE.cpp b/modules/tracking/test/test_trackerSRE.cpp new file mode 100644 index 000000000..fc9e0ed9d --- /dev/null +++ b/modules/tracking/test/test_trackerSRE.cpp @@ -0,0 +1,520 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "test_precomp.hpp" +#include "opencv2/tracking.hpp" +#include + +using namespace cv; +using namespace testing; +using namespace std; + +#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > > +#define GET_PARAM(k) std::tr1::get< k >(GetParam()) +#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") +#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50) +#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1) + +#define SPATIAL_SHIFTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10 , 11, 12) + +const string TRACKING_DIR = "tracking"; +const string FOLDER_IMG = "data"; + +/* + * The Evaluation Methodologies are partially based on: + * ==================================================================================================================== + * [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013 + * + */ + +//Robustness Evaluation, see [OTB] chapter 4. SRE Spatial robustness evaluation +class TrackerSRETest +{ + public: + enum + { + DISTANCE = 1, // test trackers based on euclidean distance + OVERLAP = 2 // test trackers based on the overlapping ratio + }; + + TrackerSRETest( const Ptr _tracker, int _testType, string _video, int _shift, float _threshold ); + virtual ~TrackerSRETest(); + virtual void run(); + string getRatioSucc() const; + + protected: + void checkDataTest(); + + void distanceTest(); + void overlapTest(); + + Ptr tracker; + int testType; + string video; + std::vector bbs; + int startFrame; + string suffix; + string prefix; + float threshold; + int shift; + float ratioSucc; + + private: + float calcDistance( Rect a, Rect b ); + float calcOverlap( Rect a, Rect b ); + std::vector splitString( std::string s, std::string delimiter ); + +}; + +TrackerSRETest::TrackerSRETest( const Ptr _tracker, int _testType, string _video, int _shift, float _threshold ) : + tracker( _tracker ), + testType( _testType ), + video( _video ), + threshold( _threshold ), + shift( _shift ) +{ + startFrame = 1; + ratioSucc = 0; +} + +TrackerSRETest::~TrackerSRETest() +{ + +} + +string TrackerSRETest::getRatioSucc() const +{ + stringstream ratio; + ratio << ratioSucc; + return ratio.str(); +} + +std::vector TrackerSRETest::splitString( std::string s, std::string delimiter ) +{ + std::vector token; + size_t pos = 0; + while ( ( pos = s.find( delimiter ) ) != std::string::npos ) + { + token.push_back( s.substr( 0, pos ) ); + s.erase( 0, pos + delimiter.length() ); + } + token.push_back( s ); + return token; +} + +float TrackerSRETest::calcDistance( Rect a, Rect b ) +{ + Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 ); + Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 ); + return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); +} + +float TrackerSRETest::calcOverlap( Rect a, Rect b ) +{ + float aArea = a.width * a.height; + float bArea = b.width * b.height; + + if( aArea < bArea ) + { + a.x -= ( b.width - a.width ) / 2; + a.y -= ( b.height - a.height ) / 2; + a.width = b.width; + a.height = b.height; + } + else + { + b.x -= ( a.width - b.width ) / 2; + b.y -= ( a.height - b.height ) / 2; + b.width = a.width; + b.height = a.height; + } + + Rect rectIntersection = a & b; + Rect rectUnion = a | b; + float iArea = rectIntersection.width * rectIntersection.height; + float uArea = rectUnion.width * rectUnion.height; + float overlap = iArea / uArea; + return overlap; +} + +void TrackerSRETest::distanceTest() +{ + Mat frame; + bool initialized = false; + + Rect currentBB = bbs.at( 0 ); + float sumDistance = 0; + int frameCounter = 0; + int frameCounterSucc = 0; + string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( ;; ) + { + c >> frame; + if( frame.empty() ) + { + break; + } + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + + float curDistance = calcDistance( currentBB, bbs.at( frameCounter ) ); + if( curDistance <= threshold ) + frameCounterSucc++; + sumDistance += curDistance; + frameCounter++; + } + + float distance = sumDistance / frameCounter; + ratioSucc = (float) frameCounterSucc / (float) frameCounter; + + if( distance > threshold ) + { + FAIL()<< "Incorrect distance: curr = " << distance << ", max = " << threshold << endl; + return; + } + +} + +void TrackerSRETest::overlapTest() +{ + Mat frame; + bool initialized = false; + Rect currentBB = bbs.at( 0 ); + float sumOverlap = 0; + string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + int frameCounter = 0; + int frameCounterSucc = 0; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( ;; ) + { + c >> frame; + if( frame.empty() ) + { + break; + } + + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + float curOverlap = calcOverlap( currentBB, bbs.at( frameCounter ) ); + if( curOverlap >= threshold ) + frameCounterSucc++; + + sumOverlap += curOverlap; + frameCounter++; + } + + float overlap = sumOverlap / frameCounter; + ratioSucc = (float) frameCounterSucc / (float) frameCounter; + + if( overlap < threshold ) + { + FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl; + return; + } +} + +void TrackerSRETest::checkDataTest() +{ + string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt"; + + ifstream gt; + //open the ground truth + gt.open( gtFile.c_str() ); + if( !gt.is_open() ) + { + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + } + string line; + int bbCounter = 0; + while ( getline( gt, line ) ) + { + vector tokens = splitString( line, "," ); + Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ); + if( tokens.size() != 4 ) + { + FAIL()<< "Incorrect ground truth file"; + } + + //apply the shift + if( bbCounter == 0 ) + { + Point center( bb.x + ( bb.width / 2 ), bb.y + ( bb.height / 2 ) ); + + int xLimit = bb.x + bb.width - 1; + int yLimit = bb.y + bb.height - 1; + + int h = 0; + int w = 0; + float ratio = 1.0; + + switch ( shift ) + { + case 1: + //center shift left + bb.x = bb.x - ceil( 0.1 * bb.width ); + break; + case 2: + //center shift right + bb.x = bb.x + ceil( 0.1 * bb.width ); + break; + case 3: + //center shift up + bb.y = bb.y - ceil( 0.1 * bb.height ); + break; + case 4: + //center shift down + bb.y = bb.y + ceil( 0.1 * bb.height ); + break; + case 5: + //corner shift top-left + bb.x = round( bb.x - ( 0.1 * bb.width ) ); + bb.y = round( bb.y - ( 0.1 * bb.height ) ); + + bb.width = xLimit - bb.x + 1; + bb.height = yLimit - bb.y + 1; + break; + case 6: + //corner shift top-right + xLimit = round( xLimit + 0.1 * bb.width ); + + bb.y = round( bb.y - ( 0.1 * bb.height ) ); + bb.width = xLimit - bb.x + 1; + bb.height = yLimit - bb.y + 1; + break; + case 7: + //corner shift bottom-left + bb.x = round( bb.x - ( 0.1 * bb.width ) ); + yLimit = round( yLimit + 0.1 * bb.height ); + + bb.width = xLimit - bb.x + 1; + bb.height = yLimit - bb.y + 1; + break; + case 8: + //corner shift bottom-right + xLimit = round( xLimit + 0.1 * bb.width ); + yLimit = round( yLimit + 0.1 * bb.height ); + + bb.width = xLimit - bb.x + 1; + bb.height = yLimit - bb.y + 1; + break; + case 9: + //scale 0.8 + ratio = 0.8; + w = ratio * bb.width; + h = ratio * bb.height; + + bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); + break; + case 10: + //scale 0.9 + ratio = 0.9; + w = ratio * bb.width; + h = ratio * bb.height; + + bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); + break; + case 11: + //scale 1.1 + ratio = 1.1; + w = ratio * bb.width; + h = ratio * bb.height; + + bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); + break; + case 12: + //scale 1.2 + ratio = 1.2; + w = ratio * bb.width; + h = ratio * bb.height; + + bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h ); + break; + default: + break; + } + } + bbs.push_back( bb ); + bbCounter++; + } + + FileStorage fs; + fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ ); + fs["start"] >> startFrame; + fs["prefix"] >> prefix; + fs["suffix"] >> suffix; + fs.release(); + +} + +void TrackerSRETest::run() +{ + srand( 1 ); + SCOPED_TRACE( "A" ); + + if( !tracker ) + { + FAIL()<< "Error in the instantiation of the tracker" << endl; + return; + } + + checkDataTest(); + + //check for failure + if( ::testing::Test::HasFatalFailure() ) + return; + + if( testType == DISTANCE ) + { + distanceTest(); + } + else if( testType == OVERLAP ) + { + overlapTest(); + } + else + { + FAIL()<< "Test type unknown" << endl; + return; + } + +} + +/****************************************************************************************\ +* Tests registrations * + \****************************************************************************************/ + +//[TESTDATA] [#SHIFT] [LOCATION ERROR THRESHOLD] +PARAM_TEST_CASE(SRE_Distance, string, int, float) +{ + string dataset; + int shift; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + shift = GET_PARAM(1); + threshold = GET_PARAM(2); + } +}; + +//[TESTDATA] [#SHIFT] [OVERLAP THRESHOLD] +PARAM_TEST_CASE(SRE_Overlap, string, int, float) +{ + string dataset; + int shift; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + shift = GET_PARAM(1); + threshold = GET_PARAM(2); + } +}; + +TEST_P(SRE_Distance, MIL) +{ + TrackerSRETest test( Tracker::create( "MIL" ), TrackerSRETest::DISTANCE, dataset, shift, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(SRE_Overlap, MIL) +{ + TrackerSRETest test( Tracker::create( "MIL" ), TrackerSRETest::OVERLAP, dataset, shift, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(SRE_Distance, Boosting) +{ + TrackerSRETest test( Tracker::create( "BOOSTING" ), TrackerSRETest::DISTANCE, dataset, shift, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(SRE_Overlap, Boosting) +{ + TrackerSRETest test( Tracker::create( "BOOSTING" ), TrackerSRETest::OVERLAP, dataset, shift, threshold ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +INSTANTIATE_TEST_CASE_P( Tracking, SRE_Distance, testing::Combine( TESTSET_NAMES, SPATIAL_SHIFTS, LOCATION_ERROR_THRESHOLD ) ); + +INSTANTIATE_TEST_CASE_P( Tracking, SRE_Overlap, testing::Combine( TESTSET_NAMES, SPATIAL_SHIFTS, OVERLAP_THRESHOLD ) ); + +/* End of file. */ diff --git a/modules/tracking/test/test_trackerTRE.cpp b/modules/tracking/test/test_trackerTRE.cpp new file mode 100644 index 000000000..c443f96ad --- /dev/null +++ b/modules/tracking/test/test_trackerTRE.cpp @@ -0,0 +1,491 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "test_precomp.hpp" +#include "opencv2/tracking.hpp" +#include +#include + +using namespace cv; +using namespace testing; +using namespace std; + +#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > > +#define GET_PARAM(k) std::tr1::get< k >(GetParam()) +#define TESTSET_NAMES testing::Values("david","dudek","faceocc2") +#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50) +#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1) +//Fixed sampling on the images sequence +#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + +const string TRACKING_DIR = "tracking"; +const string FOLDER_IMG = "data"; +const string FOLDER_OMIT_INIT = "initOmit"; + +/* + * The Evaluation Methodologies are partially based on: + * ==================================================================================================================== + * [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013 + * + */ + +//Robustness Evaluation, see [OTB] chapter 4. temporal robustness evaluation +//each sequence is partitioned into 10 (fixed) segments, slight change respect to [OTB] +class TrackerTRETest +{ + public: + enum + { + DISTANCE = 1, // test trackers based on euclidean distance + OVERLAP = 2 // test trackers based on the overlapping ratio + }; + + TrackerTRETest( const Ptr _tracker, int _testType, string _video, float _threshold, int _segmentIdx ); + virtual ~TrackerTRETest(); + virtual void run(); + string getRatioSucc() const; + + protected: + void checkDataTest(); + + void distanceTest(); + void overlapTest(); + + Ptr tracker; + int testType; + string video; + std::vector bbs; + int gtStartFrame; + int startFrame; + int endFrame; + string suffix; + string prefix; + float threshold; + int segmentIdx; + vector validSequence; + float ratioSucc; + + private: + float calcDistance( Rect a, Rect b ); + float calcOverlap( Rect a, Rect b ); + std::vector splitString( std::string s, std::string delimiter ); + +}; + +TrackerTRETest::TrackerTRETest( const Ptr _tracker, int _testType, string _video, float _threshold, int _segmentIdx ) : + tracker( _tracker ), + testType( _testType ), + video( _video ), + threshold( _threshold ), + segmentIdx( _segmentIdx ) +{ + startFrame = 1; + endFrame = 1; + gtStartFrame = 1; + ratioSucc = 0; +} + +TrackerTRETest::~TrackerTRETest() +{ + +} + +string TrackerTRETest::getRatioSucc() const +{ + stringstream ratio; + ratio << ratioSucc; + return ratio.str(); +} + +std::vector TrackerTRETest::splitString( std::string s, std::string delimiter ) +{ + std::vector token; + size_t pos = 0; + while ( ( pos = s.find( delimiter ) ) != std::string::npos ) + { + token.push_back( s.substr( 0, pos ) ); + s.erase( 0, pos + delimiter.length() ); + } + token.push_back( s ); + return token; +} + +float TrackerTRETest::calcDistance( Rect a, Rect b ) +{ + Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 ); + Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 ); + return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) ); +} + +float TrackerTRETest::calcOverlap( Rect a, Rect b ) +{ + float aArea = a.width * a.height; + float bArea = b.width * b.height; + + if( aArea < bArea ) + { + a.x -= ( b.width - a.width ) / 2; + a.y -= ( b.height - a.height ) / 2; + a.width = b.width; + a.height = b.height; + } + else + { + b.x -= ( a.width - b.width ) / 2; + b.y -= ( a.height - b.height ) / 2; + b.width = a.width; + b.height = a.height; + } + + Rect rectIntersection = a & b; + Rect rectUnion = a | b; + float iArea = rectIntersection.width * rectIntersection.height; + float uArea = rectUnion.width * rectUnion.height; + float overlap = iArea / uArea; + return overlap; +} + +void TrackerTRETest::distanceTest() +{ + Mat frame; + bool initialized = false; + + int fc = ( startFrame - gtStartFrame ); + + Rect currentBB = bbs.at( fc ); + float sumDistance = 0; + string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + int frameTotal = 0; + int frameTotalSucc = 0; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) + { + c >> frame; + if( frame.empty() ) + { + break; + } + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + + float curDistance = calcDistance( currentBB, bbs.at( fc ) ); + if( curDistance <= threshold ) + frameTotalSucc++; + sumDistance += curDistance; + + fc++; + frameTotal++; + } + + float distance = sumDistance / ( fc - ( startFrame - gtStartFrame ) ); + ratioSucc = (float) frameTotalSucc / (float) frameTotal; + + if( distance > threshold ) + { + FAIL()<< "Incorrect distance: curr = " << distance << ", min = " << threshold << endl; + return; + } + +} + +void TrackerTRETest::overlapTest() +{ + Mat frame; + bool initialized = false; + + int fc = ( startFrame - gtStartFrame ); + Rect currentBB = bbs.at( fc ); + float sumOverlap = 0; + string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG; + + int frameTotal = 0; + int frameTotalSucc = 0; + + VideoCapture c; + c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ); + c.set( CAP_PROP_POS_FRAMES, startFrame ); + + for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ ) + { + c >> frame; + if( frame.empty() ) + { + break; + } + + if( !initialized ) + { + if( !tracker->init( frame, currentBB ) ) + { + FAIL()<< "Could not initialize tracker" << endl; + return; + } + initialized = true; + } + else if( initialized ) + { + if( frameCounter >= (int) bbs.size() ) + break; + tracker->update( frame, currentBB ); + } + float curOverlap = calcOverlap( currentBB, bbs.at( fc ) ); + if( curOverlap >= threshold ) + frameTotalSucc++; + + sumOverlap += curOverlap; + frameTotal++; + fc++; + } + + float overlap = sumOverlap / ( fc - ( startFrame - gtStartFrame ) ); + ratioSucc = (float) frameTotalSucc / (float) frameTotal; + + if( overlap < threshold ) + { + FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl; + return; + } +} + +void TrackerTRETest::checkDataTest() +{ + + FileStorage fs; + fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ ); + fs["start"] >> startFrame; + fs["prefix"] >> prefix; + fs["suffix"] >> suffix; + fs.release(); + + string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt"; + ifstream gt; + //open the ground truth + gt.open( gtFile.c_str() ); + if( !gt.is_open() ) + { + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + } + string line; + int bbCounter = 0; + while ( getline( gt, line ) ) + { + bbCounter++; + } + gt.close(); + + int seqLength = bbCounter; + for ( int i = startFrame; i < seqLength; i++ ) + { + validSequence.push_back( i ); + } + + //exclude from the images sequence, the frames where the target is occluded or out of view + string omitFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_OMIT_INIT + "/" + video + ".txt"; + ifstream omit; + omit.open( omitFile.c_str() ); + if( omit.is_open() ) + { + string omitLine; + while ( getline( omit, omitLine ) ) + { + vector tokens = splitString( omitLine, " " ); + int s_start = atoi( tokens.at( 0 ).c_str() ); + int s_end = atoi( tokens.at( 1 ).c_str() ); + for ( int k = s_start; k <= s_end; k++ ) + { + std::vector::iterator position = std::find( validSequence.begin(), validSequence.end(), k ); + if( position != validSequence.end() ) + validSequence.erase( position ); + } + } + } + omit.close(); + gtStartFrame = startFrame; + //compute the start and the and for each segment + int segmentLength = sizeof ( SEGMENTS)/sizeof(int); + int numFrame = validSequence.size() / segmentLength; + startFrame += ( segmentIdx - 1 ) * numFrame; + endFrame = startFrame + numFrame; + + ifstream gt2; + //open the ground truth + gt2.open( gtFile.c_str() ); + if( !gt2.is_open() ) + { + FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl; + } + string line2; + int bbCounter2 = 0; + while ( getline( gt2, line2 ) ) + { + vector tokens = splitString( line2, "," ); + Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ); + if( tokens.size() != 4 ) + { + FAIL()<< "Incorrect ground truth file"; + } + + bbs.push_back( bb ); + bbCounter2++; + } + gt2.close(); + + if( segmentIdx == ( sizeof ( SEGMENTS)/sizeof(int) ) ) + endFrame = bbs.size(); + +} + +void TrackerTRETest::run() +{ + srand( 1 ); + SCOPED_TRACE( "A" ); + + if( !tracker ) + { + FAIL()<< "Error in the instantiation of the tracker" << endl; + return; + } + + checkDataTest(); + + //check for failure + if( ::testing::Test::HasFatalFailure() ) + return; + + if( testType == DISTANCE ) + { + distanceTest(); + } + else if( testType == OVERLAP ) + { + overlapTest(); + } + else + { + FAIL()<< "Test type unknown" << endl; + return; + } + +} + +/****************************************************************************************\ +* Tests registrations * + \****************************************************************************************/ + +//[TESTDATA] [#SEGMENT] [LOCATION ERROR THRESHOLD] +PARAM_TEST_CASE(TRE_Distance, string, int, float) +{ + int segment; + string dataset; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + segment = GET_PARAM(1); + threshold = GET_PARAM(2); + } +}; + +//[TESTDATA] [#SEGMENT] [OVERLAP THRESHOLD] +PARAM_TEST_CASE(TRE_Overlap, string, int, float) +{ + int segment; + string dataset; + float threshold; + virtual void SetUp() + { + dataset = GET_PARAM(0); + segment = GET_PARAM(1); + threshold = GET_PARAM(2); + } +}; + +TEST_P(TRE_Distance, MIL) +{ + TrackerTRETest test( Tracker::create( "MIL" ), TrackerTRETest::DISTANCE, dataset, threshold, segment ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(TRE_Overlap, MIL) +{ + TrackerTRETest test( Tracker::create( "MIL" ), TrackerTRETest::OVERLAP, dataset, threshold, segment ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(TRE_Distance, Boosting) +{ + TrackerTRETest test( Tracker::create( "BOOSTING" ), TrackerTRETest::DISTANCE, dataset, threshold, segment ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +TEST_P(TRE_Overlap, Boosting) +{ + TrackerTRETest test( Tracker::create( "BOOSTING" ), TrackerTRETest::OVERLAP, dataset, threshold, segment ); + test.run(); + RecordProperty( "ratioSuccess", test.getRatioSucc() ); +} + +INSTANTIATE_TEST_CASE_P( Tracking, TRE_Distance, testing::Combine( TESTSET_NAMES, SEGMENTS, LOCATION_ERROR_THRESHOLD ) ); + +INSTANTIATE_TEST_CASE_P( Tracking, TRE_Overlap, testing::Combine( TESTSET_NAMES, SEGMENTS, OVERLAP_THRESHOLD ) ); + +/* End of file. */