From 0d191355932070d60c91a2e9772328bcd00a9944 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 26 Jul 2015 23:18:52 +0900 Subject: [PATCH 01/32] Added Multi-tracker functionality and example 1. Multi-tracker classes (multiTracker.cpp) 2. Multi-tracker example (multiTracker_test.cpp) 3. Fixed a rare bug (OpenCL runtime error) --- .../include/opencv2/tracking/tracker.hpp | 24 ++ .../tracking/samples/multiTracker_test.cpp | 227 ++++++++++++++++++ modules/tracking/src/multiTracker.cpp | 49 ++++ modules/tracking/src/tldDetector.cpp | 2 +- 4 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 modules/tracking/samples/multiTracker_test.cpp create mode 100644 modules/tracking/src/multiTracker.cpp diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 09ef94f66..326daad86 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -49,6 +49,7 @@ #include "onlineBoosting.hpp" #include + #define BOILERPLATE_CODE(name,classname) \ static Ptr createTracker(const classname::Params ¶meters=classname::Params());\ virtual ~classname(){}; @@ -576,6 +577,7 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm Ptr model; }; + /************************************ Specific TrackerStateEstimator Classes ************************************/ /** @brief TrackerStateEstimator based on Boosting @@ -1243,6 +1245,28 @@ class CV_EXPORTS_W TrackerKCF : public Tracker BOILERPLATE_CODE("KCF",TrackerKCF); }; +/************************************ Multi-Tracker Classes ************************************/ + +class CV_EXPORTS_W MultiTracker +{ +public: + + bool addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name); + + bool update(const Mat& image); + + int targetNum = 0; + std::vector > trackers; + std::vector boundingBoxes; + std::vector colors; +}; + +class CV_EXPORTS_W MultiTrackerTLD : public MultiTracker +{ +public: + bool update(const Mat& image); +}; + //! @} } /* namespace cv */ diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp new file mode 100644 index 000000000..ac3c37382 --- /dev/null +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -0,0 +1,227 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +#define NUM_TEST_FRAMES 100 +#define TEST_VIDEO_INDEX 7 //TLD Dataset Video Index from 1-10 +//#define RECORD_VIDEO_FLG + +static Mat image; +static bool paused; +static bool selectObject = false; +static bool startSelection = false; +Rect2d boundingBox; + +static void onMouse(int event, int x, int y, int, void*) +{ + if (!selectObject) + { + switch (event) + { + case EVENT_LBUTTONDOWN: + //set origin of the bounding box + startSelection = true; + boundingBox.x = x; + boundingBox.y = y; + boundingBox.width = boundingBox.height = 0; + break; + case EVENT_LBUTTONUP: + //sei with and height of the bounding box + boundingBox.width = std::abs(x - boundingBox.x); + boundingBox.height = std::abs(y - boundingBox.y); + paused = false; + selectObject = true; + break; + case EVENT_MOUSEMOVE: + + if (startSelection && !selectObject) + { + //draw the bounding box + Mat currentFrame; + image.copyTo(currentFrame); + rectangle(currentFrame, Point((int)boundingBox.x, (int)boundingBox.y), Point(x, y), Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", currentFrame); + } + break; + } + } +} + +int main() +{ + // + // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" + // + char* tracker_algorithm_name = "TLD"; + + Mat frame; + paused = false; + namedWindow("Tracking API", 0); + setMouseCallback("Tracking API", onMouse, 0); + + MultiTrackerTLD mt; + + //Get the first frame + ////Open the capture + // VideoCapture cap(0); + // if( !cap.isOpened() ) + // { + // cout << "Video stream error"; + // return; + // } + //cap >> frame; + + //From TLD dataset + selectObject = true; + Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/TLD_dataset"); + Rect2d boundingBox2; + boundingBox2.x = 280; + boundingBox2.y = 60; + boundingBox2.width = 40; + boundingBox2.height = 60; + + frame = tld::tld_getNextDatasetFrame(); + frame.copyTo(image); + + // Setup output video +#ifdef RECORD_VIDEO_FLG + String outputFilename = "test.avi"; + VideoWriter outputVideo; + outputVideo.open(outputFilename, -1, 30, Size(image.cols, image.rows)); + + if (!outputVideo.isOpened()) + { + std::cout << "!!! Output video could not be opened" << std::endl; + getchar(); + return; + } +#endif + + rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", image); + + + bool initialized = false; + int frameCounter = 0; + + //Time measurment + int64 e3 = getTickCount(); + + for (;;) + { + //Time measurment + int64 e1 = getTickCount(); + //Frame num + frameCounter++; + if (frameCounter == NUM_TEST_FRAMES) break; + + char c = (char)waitKey(2); + if (c == 'q' || c == 27) + break; + if (c == 'p') + paused = !paused; + + if (!paused) + { + //cap >> frame; + frame = tld::tld_getNextDatasetFrame(); + if (frame.empty()) + { + break; + } + frame.copyTo(image); + + if (selectObject) + { + if (!initialized) + { + //initializes the tracker + mt.addTarget(frame, boundingBox1, tracker_algorithm_name); + rectangle(image, boundingBox1, mt.colors[0], 2, 1); + + + mt.addTarget(frame, boundingBox2, tracker_algorithm_name); + rectangle(image, boundingBox2, mt.colors[1], 2, 1); + initialized = true; + } + else + { + //updates the tracker + if (mt.update(frame)) + for (int i=0; i < mt.targetNum; i++) + rectangle(image, mt.boundingBoxes[i], mt.colors[i], 2, 1); + } + } + imshow("Tracking API", image); + +#ifdef RECORD_VIDEO_FLG + outputVideo << image; +#endif + + + //Time measurment + int64 e2 = getTickCount(); + double t1 = (e2 - e1) / getTickFrequency(); + cout << frameCounter << "\tframe : " << t1 * 1000.0 << "ms" << endl; + + waitKey(0); + } + } + + //Time measurment + int64 e4 = getTickCount(); + double t2 = (e4 - e3) / getTickFrequency(); + cout << "Average Time for Frame: " << t2 * 1000.0 / frameCounter << "ms" << endl; + cout << "Average FPS: " << 1.0 / t2*frameCounter << endl; + + + waitKey(0); + + return 0; +} \ No newline at end of file diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp new file mode 100644 index 000000000..0ce6e55ca --- /dev/null +++ b/modules/tracking/src/multiTracker.cpp @@ -0,0 +1,49 @@ +#include + +namespace cv +{ + //Multitracker + bool MultiTracker::addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name) + { + Ptr tracker = Tracker::create(tracker_algorithm_name); + if (tracker == NULL) + return false; + + if (!tracker->init(image, boundingBox)) + return false; + + //Add BB of target + boundingBoxes.push_back(boundingBox); + + //Add Tracker to stack + trackers.push_back(tracker); + + //Assign a random color to target + colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + + //Target counter + targetNum++; + + return true; + } + + bool MultiTracker::update(const Mat& image) + { + for (int i = 0; i < trackers.size(); i++) + if (!trackers[i]->update(image, boundingBoxes[i])) + return false; + + return true; + } + + //Multitracker TLD + /*Optimized update method for TLD Multitracker */ + bool MultiTrackerTLD::update(const Mat& image) + { + for (int i = 0; i < trackers.size(); i++) + if (!trackers[i]->update(image, boundingBoxes[i])) + return false; + + return true; + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index cbb783ece..900761bef 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -219,7 +219,7 @@ namespace cv // 2 -> Pos&Neg size_t globSize = 2 * numOfPatches*MAX_EXAMPLES_IN_MODEL; //e3 = getTickCount(); - if (!k.run(1, &globSize, NULL, false)) + if (!k.run(1, &globSize, NULL, true)) printf("Kernel Run Error!!!"); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; From 6d74b7ff4a791150410ccb842cf6e3d2a9030f2a Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 27 Jul 2015 15:41:19 +0900 Subject: [PATCH 02/32] Added optimization to Multi-target TLD update --- .../include/opencv2/tracking/tracker.hpp | 2 +- modules/tracking/src/multiTracker.cpp | 150 +++++++++++++++++- modules/tracking/src/tldTracker.hpp | 1 - 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 326daad86..a9d104a0b 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -565,7 +565,7 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm virtual void read( const FileNode& fn )=0; virtual void write( FileStorage& fs ) const=0; - protected: + public: virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0ce6e55ca..92c7f51b1 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,4 +1,4 @@ -#include +#include "tldTracker.hpp" namespace cv { @@ -35,14 +35,156 @@ namespace cv return true; } - + //Multitracker TLD /*Optimized update method for TLD Multitracker */ bool MultiTrackerTLD::update(const Mat& image) { - for (int i = 0; i < trackers.size(); i++) - if (!trackers[i]->update(image, boundingBoxes[i])) + + for (int k = 0; k < trackers.size(); k++) + { + //Set current target(tracker) parameters + Rect2d boundingBox = boundingBoxes[k]; + Ptr tracker = (Ptr)static_cast> (trackers[k]); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + double scale = data->getScale(); + + + Mat image_gray, image_blurred, imageForDetector; + cvtColor(image, image_gray, COLOR_BGR2GRAY); + + if (scale > 1.0) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + + data->frameNum++; + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + std::vector detectorResults; + //best overlap around 92% + + std::vector candidates; + std::vector candidatesRes; + bool trackerNeedsReInit = false; + bool DETECT_FLG = false; + for (int i = 0; i < 2; i++) + { + Rect2d tmpCandid = boundingBox; + + if (i == 1) + { + if (ocl::haveOpenCL()) + DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + else + DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + } + + if (((i == 0) && !data->failedLastTime && tracker->trackerProxy->update(image, tmpCandid)) || (DETECT_FLG)) + { + candidates.push_back(tmpCandid); + if (i == 0) + tld::resample(image_gray, tmpCandid, standardPatch); + else + tld::resample(imageForDetector, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + } + else + { + if (i == 0) + trackerNeedsReInit = true; + } + } + + std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + + //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); + //for( int i = 0; i < (int)candidatesRes.size(); i++ ) + //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); + //data->printme(); + //tldModel->printme(stdout); + + if (it == candidatesRes.end()) + { + data->confident = false; + data->failedLastTime = true; return false; + } + else + { + boundingBox = candidates[it - candidatesRes.begin()]; + boundingBoxes[k] = boundingBox; + data->failedLastTime = false; + if (trackerNeedsReInit || it != candidatesRes.begin()) + tracker->trackerProxy->init(image, boundingBox); + } + +#if 1 + if (it != candidatesRes.end()) + { + tld::resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); + //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) + //dfprintf((stderr, "detector WON\n")); + } + else + { + //dfprintf((stderr, "%d x x\n", data->frameNum)); + } +#endif + + if (*it > tld::CORE_THRESHOLD) + data->confident = true; + + if (data->confident) + { + tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, tracker->params, data->getMinSize()); + tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, tracker->params); + std::vector > examplesForModel, examplesForEnsemble; + examplesForModel.reserve(100); examplesForEnsemble.reserve(100); + int negRelabeled = 0; + for (int i = 0; i < (int)detectorResults.size(); i++) + { + bool expertResult; + if (detectorResults[i].isObject) + { + expertResult = nExpert(detectorResults[i].rect); + if (expertResult != detectorResults[i].isObject) + negRelabeled++; + } + else + { + expertResult = pExpert(detectorResults[i].rect); + } + + detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); + detectorResults[i].isObject = expertResult; + } + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + //dprintf(("%d relabeled by nExpert\n", negRelabeled)); + pExpert.additionalExamples(examplesForModel, examplesForEnsemble); + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); + examplesForModel.clear(); examplesForEnsemble.clear(); + nExpert.additionalExamples(examplesForModel, examplesForEnsemble); + + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); + } + else + { +#ifdef CLOSED_LOOP + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); +#endif + } + + + } return true; } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index e04926d5d..40de754b7 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -128,7 +128,6 @@ public: void read(const FileNode& fn); void write(FileStorage& fs) const; -protected: class Pexpert { public: From ae1669918f7019cfd01f13b86be522bccea5e385 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:44:14 +0900 Subject: [PATCH 03/32] Added VOT 2015 dataset support http://www.votchallenge.net/vot2015/dataset.html --- .../include/opencv2/tracking/tldDataset.hpp | 2 +- modules/tracking/src/tldDataset.cpp | 148 +++++++++++------- 2 files changed, 95 insertions(+), 55 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index 6d21a7d48..d31919ab5 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -48,7 +48,7 @@ namespace cv { namespace tld { - CV_EXPORTS cv::Rect2d tld_InitDataset(int datasetInd, const char* rootPath = "TLD_dataset"); + CV_EXPORTS cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath = "TLD_dataset", int datasetInd = 0); CV_EXPORTS cv::Mat tld_getNextDatasetFrame(); } } diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 8432fc272..ddb241a83 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -48,67 +48,105 @@ namespace cv char tldRootPath[100]; int frameNum = 0; bool flagPNG = false; + bool flagVOT = false; - cv::Rect2d tld_InitDataset(int datasetInd,const char* rootPath) + //TLD Dataset Parameters + char* tldFolderName[10] = { + "01_david", + "02_jumping", + "03_pedestrian1", + "04_pedestrian2", + "05_pedestrian3", + "06_car", + "07_motocross", + "08_volkswagen", + "09_carchase", + "10_panda" + }; + char* votFolderName[60] = { + "bag", "ball1", "ball2", "basketball", "birds1", "birds2", "blanket", "bmx", "bolt1", "bolt2", + "book", "butterfly", "car1", "car2", "crossing", "dinosaur", "fernando", "fish1", "fish2", "fish3", + "fish4", "girl", "glove", "godfather", "graduate", "gymnastics1", "gymnastics2 ", "gymnastics3", "gymnastics4", "hand", + "handball1", "handball2", "helicopter", "iceskater1", "iceskater2", "leaves", "marching", "matrix", "motocross1", "motocross2", + "nature", "octopus", "pedestrian1", "pedestrian2", "rabbit", "racing", "road", "shaking", "sheep", "singer1", + "singer2", "singer3", "soccer1", "soccer2", "soldier", "sphere", "tiger", "traffic", "tunnel", "wiper" + }; + + Rect2d tldInitBB[10] = { + Rect2d(165, 183, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(337, 219, 54, 37), Rect2d(58, 100, 27, 22) + }; + Rect2d votInitBB[60] = { + Rect2d(142, 125, 90, 39), Rect2d(490, 400, 40, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 100), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + }; + + int tldFrameOffset[10] = { 100, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + int votFrameOffset[60] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + bool tldFlagPNG[10] = { 0, 0, 0, 0, 0, 0, 1, 0, 0, 0 }; + bool votFlagPNG[60] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath, int datasetInd) { char* folderName = (char *)""; int x, y, w, h; - flagPNG = false; - frameNum = 1; + //Index range + // 1-10 TLD Dataset + // 1-60 VOT 2015 Dataset + int id = videoInd - 1; - if (datasetInd == 1) { - folderName = (char *)"01_david"; - x = 165, y = 83; - w = 51; h = 54; - frameNum = 100; - } - if (datasetInd == 2) { - folderName = (char *)"02_jumping"; - x = 147, y = 110; - w = 33; h = 32; - } - if (datasetInd == 3) { - folderName = (char *)"03_pedestrian1"; - x = 47, y = 51; - w = 21; h = 36; - } - if (datasetInd == 4) { - folderName = (char *)"04_pedestrian2"; - x = 130, y = 134; - w = 21; h = 53; - } - if (datasetInd == 5) { - folderName = (char *)"05_pedestrian3"; - x = 154, y = 102; - w = 24; h = 52; - } - if (datasetInd == 6) { - folderName = (char *)"06_car"; - x = 142, y = 125; - w = 90; h = 39; - } - if (datasetInd == 7) { - folderName = (char *)"07_motocross"; - x = 290, y = 43; - w = 23; h = 40; - flagPNG = true; - } - if (datasetInd == 8) { - folderName = (char *)"08_volkswagen"; - x = 273, y = 77; - w = 27; h = 25; - } - if (datasetInd == 9) { - folderName = (char *)"09_carchase"; - x = 145, y = 84; - w = 54; h = 37; - } - if (datasetInd == 10){ - folderName = (char *)"10_panda"; - x = 58, y = 100; - w = 27; h = 22; + if (datasetInd == 0) + { + folderName = tldFolderName[id]; + x = tldInitBB[id].x; + y = tldInitBB[id].y; + w = tldInitBB[id].width; + h = tldInitBB[id].height; + frameNum = tldFrameOffset[id]; + flagPNG = tldFlagPNG[id]; + flagVOT = false; } + if (datasetInd == 1) + { + folderName = votFolderName[id]; + x = votInitBB[id].x; + y = votInitBB[id].y; + w = votInitBB[id].width; + h = votInitBB[id].height; + frameNum = votFrameOffset[id]; + flagPNG = votFlagPNG[id]; + flagVOT = true; + } strcpy(tldRootPath, rootPath); strcat(tldRootPath, "\\"); @@ -124,6 +162,8 @@ namespace cv char numStr[10]; strcpy(fullPath, tldRootPath); strcat(fullPath, "\\"); + if (flagVOT) + strcat(fullPath, "000"); if (frameNum < 10) strcat(fullPath, "0000"); else if (frameNum < 100) strcat(fullPath, "000"); else if (frameNum < 1000) strcat(fullPath, "00"); From 162a65854e69deef708c4146a1e38808d53b48d9 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:46:33 +0900 Subject: [PATCH 04/32] Debug through candidates display --- modules/tracking/src/tldDetector.cpp | 14 ++++++++++++-- modules/tracking/src/tldDetector.hpp | 14 ++++++++------ modules/tracking/src/tldTracker.cpp | 22 ++++++++++++++++++---- modules/tracking/src/tldTracker.hpp | 6 ------ 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 900761bef..0443f46cb 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -491,6 +491,10 @@ namespace cv continue; varBuffer.push_back(Point(dx * i, dy * j)); varScaleIDs.push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, initSize.width*curScale, initSize.height*curScale)); } } scaleID++; @@ -520,6 +524,9 @@ namespace cv //t = (e2 - e1) / getTickFrequency()*1000.0; //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); + //printf("varBuffer: %d\n", varBuffer.size()); + //printf("ensBuffer: %d\n", ensBuffer.size()); + //NN classification //e1 = getTickCount(); for (int i = 0; i < (int)ensBuffer.size(); i++) @@ -561,8 +568,11 @@ namespace cv if (maxSc < 0) return false; - res = maxScRect; - return true; + else + { + res = maxScRect; + return true; + } } bool TLDDetector::ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize) diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 3e1189592..4dc433607 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -66,6 +66,8 @@ namespace cv static const cv::Size GaussBlurKernelSize(3, 3); + + class TLDDetector { public: @@ -79,6 +81,7 @@ namespace cv double ocl_Sc(const Mat_& patch); void ocl_batchSrSc(const Mat_& patches, double *resultSr, double *resultSc, int numOfPatches); + std::vector debugStack[10]; std::vector classifiers; Mat *posExp, *negExp; int *posNum, *negNum; @@ -93,15 +96,14 @@ namespace cv bool isObject, shouldBeIntegrated; }; bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); - bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); - protected: - - + bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); friend class MyMouseCallbackDEBUG; - void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } - inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); + static void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } + static inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); }; + + } } diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 0f1e2f78e..2af2d8e8e 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -45,6 +45,13 @@ namespace cv { + TrackerTLD::Params::Params(){} + + void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} + + void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} + + Ptr TrackerTLD::createTracker(const TrackerTLD::Params ¶meters) { return Ptr(new tld::TrackerTLDImpl(parameters)); @@ -112,7 +119,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); std::vector detectorResults; //best overlap around 92% - std::vector candidates; std::vector candidatesRes; bool trackerNeedsReInit = false; @@ -123,12 +129,11 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { - if (ocl::haveOpenCL()) + if (!ocl::haveOpenCL()) DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); else DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); } - if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) { candidates.push_back(tmpCandid); @@ -144,7 +149,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) trackerNeedsReInit = true; } } - std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); @@ -230,6 +234,16 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) #endif } + + //Debug display candidates after Variance Filter + //////////////////////////////////////////////// + Mat tmpImg = image; + for (int i = 0; i < tldModel->detector->debugStack[0].size(); i++) + //rectangle(tmpImg, tldModel->detector->debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); + tldModel->detector->debugStack[0].clear(); + tmpImg.copyTo(image); + + //////////////////////////////////////////////// return true; } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 40de754b7..4a5610de3 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -52,12 +52,6 @@ namespace cv { -TrackerTLD::Params::Params(){} - -void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} - -void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} - namespace tld { class TrackerProxy From 19ea56448677f8004e5251d61be1d234b2640ae8 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:49:32 +0900 Subject: [PATCH 05/32] Improved VF optimization + Added EC optimization for MO-TLD --- .../include/opencv2/tracking/tracker.hpp | 2 +- .../tracking/samples/multiTracker_test.cpp | 25 +- modules/tracking/src/multiTracker.cpp | 385 +++++++++++++++--- .../tracking/src/tldEnsembleClassifier.hpp | 2 +- 4 files changed, 344 insertions(+), 70 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index a9d104a0b..54adbe301 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1264,7 +1264,7 @@ public: class CV_EXPORTS_W MultiTrackerTLD : public MultiTracker { public: - bool update(const Mat& image); + bool update_opt(const Mat& image); }; //! @} diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index ac3c37382..6831c1f88 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -49,7 +49,7 @@ using namespace std; using namespace cv; #define NUM_TEST_FRAMES 100 -#define TEST_VIDEO_INDEX 7 //TLD Dataset Video Index from 1-10 +#define TEST_VIDEO_INDEX 15 //TLD Dataset Video Index from 1-10 for TLD and 1-60 for VOT //#define RECORD_VIDEO_FLG static Mat image; @@ -119,12 +119,12 @@ int main() //From TLD dataset selectObject = true; - Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/TLD_dataset"); + Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/VOT 2015", 1); Rect2d boundingBox2; - boundingBox2.x = 280; - boundingBox2.y = 60; - boundingBox2.width = 40; - boundingBox2.height = 60; + boundingBox2.x = 470; + boundingBox2.y = 500; + boundingBox2.width = 50; + boundingBox2.height = 100; frame = tld::tld_getNextDatasetFrame(); frame.copyTo(image); @@ -140,6 +140,7 @@ int main() std::cout << "!!! Output video could not be opened" << std::endl; getchar(); return; + } #endif @@ -193,12 +194,14 @@ int main() else { //updates the tracker - if (mt.update(frame)) - for (int i=0; i < mt.targetNum; i++) - rectangle(image, mt.boundingBoxes[i], mt.colors[i], 2, 1); + if (mt.update_opt(frame)) + { + for (int i = 0; i < mt.targetNum; i++) + rectangle(frame, mt.boundingBoxes[i], mt.colors[i], 2, 1); + } } } - imshow("Tracking API", image); + imshow("Tracking API", frame); #ifdef RECORD_VIDEO_FLG outputVideo << image; @@ -210,7 +213,7 @@ int main() double t1 = (e2 - e1) / getTickFrequency(); cout << frameCounter << "\tframe : " << t1 * 1000.0 << "ms" << endl; - waitKey(0); + //waitKey(0); } } diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 92c7f51b1..0dd3aa4a8 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,4 +1,4 @@ -#include "tldTracker.hpp" +#include "multiTracker.hpp" namespace cv { @@ -29,75 +29,104 @@ namespace cv bool MultiTracker::update(const Mat& image) { + printf("Naive-Loop MO-TLD Update....\n"); for (int i = 0; i < trackers.size(); i++) if (!trackers[i]->update(image, boundingBoxes[i])) return false; return true; } - + //Multitracker TLD /*Optimized update method for TLD Multitracker */ - bool MultiTrackerTLD::update(const Mat& image) + bool MultiTrackerTLD::update_opt(const Mat& image) { - - for (int k = 0; k < trackers.size(); k++) - { - //Set current target(tracker) parameters - Rect2d boundingBox = boundingBoxes[k]; - Ptr tracker = (Ptr)static_cast> (trackers[k]); - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); - Ptr data = tracker->data; - double scale = data->getScale(); + printf("Optimized MO-TLD Update....\n"); + //Get parameters from first object + //Set current target(tracker) parameters + Rect2d boundingBox = boundingBoxes[0]; + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + double scale = data->getScale(); - Mat image_gray, image_blurred, imageForDetector; - cvtColor(image, image_gray, COLOR_BGR2GRAY); + Mat image_gray, image_blurred, imageForDetector; + cvtColor(image, image_gray, COLOR_BGR2GRAY); - if (scale > 1.0) - resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); - else - imageForDetector = image_gray; - GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + if (scale > 1.0) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + + //best overlap around 92% + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + + std::vector> detectorResults(targetNum); + std::vector> candidates(targetNum); + std::vector> candidatesRes(targetNum); + std::vector tmpCandidates(targetNum); + std::vector detect_flgs(targetNum); + std::vector trackerNeedsReInit(targetNum); + + bool DETECT_FLG = false; + + //printf("%d\n", targetNum); + //Detect all + for (int k = 0; k < targetNum; k++) + tmpCandidates[k] = boundingBoxes[k]; + //if (ocl::haveOpenCL()) + detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); + //else + //DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + + //printf("BOOOLZZZ %d\n", detect_flgs[0]); + //printf("BOOOLXXX %d\n", detect_flgs[1]); + for (int k = 0; k < targetNum; k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + /////// data->frameNum++; - Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); - std::vector detectorResults; - //best overlap around 92% - - std::vector candidates; - std::vector candidatesRes; - bool trackerNeedsReInit = false; - bool DETECT_FLG = false; + for (int i = 0; i < 2; i++) { - Rect2d tmpCandid = boundingBox; + Rect2d tmpCandid = boundingBoxes[k]; - if (i == 1) + //if (i == 1) { - if (ocl::haveOpenCL()) - DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - else - DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + DETECT_FLG = detect_flgs[k]; + tmpCandid = tmpCandidates[k]; } if (((i == 0) && !data->failedLastTime && tracker->trackerProxy->update(image, tmpCandid)) || (DETECT_FLG)) { - candidates.push_back(tmpCandid); + candidates[k].push_back(tmpCandid); if (i == 0) tld::resample(image_gray, tmpCandid, standardPatch); else tld::resample(imageForDetector, tmpCandid, standardPatch); - candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + candidatesRes[k].push_back(tldModel->detector->Sc(standardPatch)); } else { if (i == 0) - trackerNeedsReInit = true; + trackerNeedsReInit[k] = true; + else + trackerNeedsReInit[k] = false; } } - - std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + //printf("CanditateRes Size: %d \n", candidatesRes[k].size()); + std::vector::iterator it = std::max_element(candidatesRes[k].begin(), candidatesRes[k].end()); //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); //for( int i = 0; i < (int)candidatesRes.size(); i++ ) @@ -105,25 +134,25 @@ namespace cv //data->printme(); //tldModel->printme(stdout); - if (it == candidatesRes.end()) + if (it == candidatesRes[k].end()) { + data->confident = false; data->failedLastTime = true; return false; } else { - boundingBox = candidates[it - candidatesRes.begin()]; - boundingBoxes[k] = boundingBox; + boundingBoxes[k] = candidates[k][it - candidatesRes[k].begin()]; data->failedLastTime = false; - if (trackerNeedsReInit || it != candidatesRes.begin()) - tracker->trackerProxy->init(image, boundingBox); + if (trackerNeedsReInit[k] || it != candidatesRes[k].begin()) + tracker->trackerProxy->init(image, boundingBoxes[k]); } #if 1 - if (it != candidatesRes.end()) + if (it != candidatesRes[k].end()) { - tld::resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + tld::resample(imageForDetector, candidates[k][it - candidatesRes[k].begin()], standardPatch); //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) //dfprintf((stderr, "detector WON\n")); @@ -139,29 +168,29 @@ namespace cv if (data->confident) { - tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, tracker->params, data->getMinSize()); - tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, tracker->params); + tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBoxes[k], tldModel->detector, tracker->params, data->getMinSize()); + tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBoxes[k], tldModel->detector, tracker->params); std::vector > examplesForModel, examplesForEnsemble; examplesForModel.reserve(100); examplesForEnsemble.reserve(100); int negRelabeled = 0; - for (int i = 0; i < (int)detectorResults.size(); i++) + for (int i = 0; i < (int)detectorResults[k].size(); i++) { bool expertResult; - if (detectorResults[i].isObject) + if (detectorResults[k][i].isObject) { - expertResult = nExpert(detectorResults[i].rect); - if (expertResult != detectorResults[i].isObject) + expertResult = nExpert(detectorResults[k][i].rect); + if (expertResult != detectorResults[k][i].isObject) negRelabeled++; } else { - expertResult = pExpert(detectorResults[i].rect); + expertResult = pExpert(detectorResults[k][i].rect); } - detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); - detectorResults[i].isObject = expertResult; + detectorResults[k][i].shouldBeIntegrated = detectorResults[k][i].shouldBeIntegrated || (detectorResults[k][i].isObject != expertResult); + detectorResults[k][i].isObject = expertResult; } - tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults[k]); //dprintf(("%d relabeled by nExpert\n", negRelabeled)); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); if (ocl::haveOpenCL()) @@ -183,9 +212,251 @@ namespace cv #endif } - - } + } + //Debug display candidates after Variance Filter + //////////////////////////////////////////////// + Mat tmpImg = image; + for (int i = 0; i < debugStack[0].size(); i++) + //rectangle(tmpImg, debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); + debugStack[0].clear(); + tmpImg.copyTo(image); + //////////////////////////////////////////////// return true; } + + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, + std::vector> &trackers) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Size initSize = tldModel->getMinSize(); + + for (int k = 0; k < trackers.size(); k++) + patches[k].clear(); + + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + Mat tmp; + int dx = initSize.width / 10, dy = initSize.height / 10; + Size2d size = img.size(); + double scale = 1.0; + int npos = 0, nneg = 0; + double maxSc = -5.0; + Rect2d maxScRect; + int scaleID; + std::vector resized_imgs, blurred_imgs; + + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + + std::vector tmpP; + std::vector tmpI; + + //int64 e1, e2; + //double t; + //e1 = getTickCount(); + + //Detection part + //Generate windows and filter by variance + scaleID = 0; + resized_imgs.push_back(img); + blurred_imgs.push_back(imgBlurred); + do + { + Mat_ intImgP, intImgP2; + tld::TLDDetector::computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2); + for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++) + { + for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++) + { + //Optimized variance calculation + int x = dx * i, + y = dy * j, + width = initSize.width, + height = initSize.height; + double p = 0, p2 = 0; + double A, B, C, D; + + A = intImgP(y, x); + B = intImgP(y, x + width); + C = intImgP(y + height, x); + D = intImgP(y + height, x + width); + p = (A + D - B - C) / (width * height); + + A = intImgP2(y, x); + B = intImgP2(y, x + width); + C = intImgP2(y + height, x); + D = intImgP2(y + height, x + width); + p2 = (A + D - B - C) / (width * height); + double windowVar = p2 - p * p; + + //Loop for on all objects + for (int k=0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + //Optimized variance calculation + bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); + + if (!varPass) + continue; + varBuffer[k].push_back(Point(dx * i, dy * j)); + varScaleIDs[k].push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); + } + } + } + scaleID++; + size.width /= tld::SCALE_STEP; + size.height /= tld::SCALE_STEP; + scale *= tld::SCALE_STEP; + resize(img, tmp, size, 0, 0, tld::DOWNSCALE_MODE); + resized_imgs.push_back(tmp); + GaussianBlur(resized_imgs[scaleID], tmp, tld::GaussBlurKernelSize, 0.0f); + blurred_imgs.push_back(tmp); + } while (size.width >= initSize.width && size.height >= initSize.height); + + + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Variance: %d\t%f\n", varBuffer.size(), t); + + //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); + + //Encsemble classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + + double ensRes = 0; + uchar* data = &blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x); + for (int x = 0; x < (int)tldModel->detector->classifiers.size(); x++) + { + int position = 0; + for (int n = 0; n < (int)tldModel->detector->classifiers[x].measurements.size(); n++) + { + position = position << 1; + if (data[tldModel->detector->classifiers[x].offset[n].x] < data[tldModel->detector->classifiers[x].offset[n].y]) + position++; + } + double posNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].x; + double negNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + continue; + else + ensRes += posNum / (posNum + negNum); + } + ensRes /= tldModel->detector->classifiers.size(); + ensRes = tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)); + + if ( ensRes <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + /* + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + */ + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); + + //printf("varBuffer 1: %d\n", varBuffer[0].size()); + //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); + + //printf("varBuffer 2: %d\n", varBuffer[1].size()); + //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); + + //NN classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + npos = 0; + nneg = 0; + maxSc = -5.0; + + for (int i = 0; i < (int)ensBuffer[k].size(); i++) + { + tld::TLDDetector::LabeledPatch labPatch; + double curScale = pow(tld::SCALE_STEP, ensScaleIDs[k][i]); + labPatch.rect = Rect2d(ensBuffer[k][i].x*curScale, ensBuffer[k][i].y*curScale, initSize.width * curScale, initSize.height * curScale); + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + + double srValue, scValue; + srValue = tldModel->detector->Sr(standardPatch); + + ////To fix: Check the paper, probably this cause wrong learning + // + labPatch.isObject = srValue > tld::THETA_NN; + labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1; + patches[k].push_back(labPatch); + // + + if (!labPatch.isObject) + { + nneg++; + continue; + } + else + { + npos++; + } + scValue = tldModel->detector->Sc(standardPatch); + if (scValue > maxSc) + { + maxSc = scValue; + maxScRect = labPatch.rect; + } + //printf("%d %f %f\n", k, srValue, scValue); + } + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); + + if (maxSc < 0) + detect_flgs[k] = false; + else + { + res[k] = maxScRect; + //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); + detect_flgs[k] = true; + } + } + } } \ No newline at end of file diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp index 97221372d..f0ec175ba 100644 --- a/modules/tracking/src/tldEnsembleClassifier.hpp +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -54,7 +54,7 @@ namespace cv double posteriorProbability(const uchar* data, int rowstep) const; double posteriorProbabilityFast(const uchar* data) const; void prepareClassifier(int rowstep); - private: + TLDEnsembleClassifier(const std::vector& meas, int beg, int end); static void stepPrefSuff(std::vector & arr, int pos, int len, int gridSize); int code(const uchar* data, int rowstep) const; From 8749c49a0084e15cfa068744ff3fbb5559716aa9 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 09:24:56 +0900 Subject: [PATCH 06/32] Added multiTracker.hpp (fix) --- modules/tracking/src/multiTracker.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 modules/tracking/src/multiTracker.hpp diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp new file mode 100644 index 000000000..a2b0a01ea --- /dev/null +++ b/modules/tracking/src/multiTracker.hpp @@ -0,0 +1,16 @@ +#ifndef OPENCV_MULTITRACKER +#define OPENCV_MULTITRACKER + +#include "tldTracker.hpp" +#include "tldUtils.hpp" +#include + +namespace cv +{ + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, + std::vector& detect_flgs, std::vector>& trackers); + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, + std::vector& detect_flgs, std::vector>& trackers); + std::vector debugStack[10]; +} +#endif \ No newline at end of file From b412fbde18555459fba3f9a85acee170d7c725aa Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:36:21 +0900 Subject: [PATCH 07/32] Fix std::vector > --- modules/tracking/include/opencv2/tracking/tracker.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 54adbe301..8144e72bd 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1256,7 +1256,7 @@ public: bool update(const Mat& image); int targetNum = 0; - std::vector > trackers; + std::vector > trackers; std::vector boundingBoxes; std::vector colors; }; From ae73daf6d012e665789c5f0086d8b80926d7dacc Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:38:31 +0900 Subject: [PATCH 08/32] Fixed some init BB --- modules/tracking/src/tldDataset.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index ddb241a83..16d5596f2 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -73,14 +73,14 @@ namespace cv }; Rect2d tldInitBB[10] = { - Rect2d(165, 183, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), + Rect2d(165, 93, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(337, 219, 54, 37), Rect2d(58, 100, 27, 22) }; Rect2d votInitBB[60] = { Rect2d(142, 125, 90, 39), Rect2d(490, 400, 40, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), - Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), + Rect2d(450, 380, 60, 60), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), - Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 100), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 120), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), From 23acf6d2cdfe2171a7da91284758d5a4c9c80044 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:43:31 +0900 Subject: [PATCH 09/32] Fixed pointers to posNum and negNum --- modules/tracking/src/tldDetector.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 0443f46cb..bfef69a45 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -131,8 +131,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devPositiveSamples), ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devNCC), - posNum, - negNum); + *posNum, + *negNum); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; @@ -208,8 +208,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devPosNCC), ocl::KernelArg::PtrWriteOnly(devNegNCC), - posNum, - negNum, + *posNum, + *negNum, numOfPatches); //e4 = getTickCount(); @@ -364,8 +364,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devPositiveSamples), ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devNCC), - posNum, - negNum); + *posNum, + *negNum); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; From 59b18dcfab667406bafaee326b0c10545187b55d Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:45:21 +0900 Subject: [PATCH 10/32] Added MO-TLD NN Classifier optimization on GPU --- .../tracking/samples/multiTracker_test.cpp | 18 +- modules/tracking/src/multiTracker.cpp | 285 ++++++++++++++++-- modules/tracking/src/tldTracker.cpp | 2 +- 3 files changed, 276 insertions(+), 29 deletions(-) diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index 6831c1f88..c202a161f 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -120,11 +120,7 @@ int main() //From TLD dataset selectObject = true; Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/VOT 2015", 1); - Rect2d boundingBox2; - boundingBox2.x = 470; - boundingBox2.y = 500; - boundingBox2.width = 50; - boundingBox2.height = 100; + Rect2d boundingBox2(470, 490, 50, 120); frame = tld::tld_getNextDatasetFrame(); frame.copyTo(image); @@ -133,13 +129,13 @@ int main() #ifdef RECORD_VIDEO_FLG String outputFilename = "test.avi"; VideoWriter outputVideo; - outputVideo.open(outputFilename, -1, 30, Size(image.cols, image.rows)); + outputVideo.open(outputFilename, -1, 15, Size(image.cols, image.rows)); if (!outputVideo.isOpened()) { std::cout << "!!! Output video could not be opened" << std::endl; getchar(); - return; + return 0; } #endif @@ -184,17 +180,17 @@ int main() { //initializes the tracker mt.addTarget(frame, boundingBox1, tracker_algorithm_name); - rectangle(image, boundingBox1, mt.colors[0], 2, 1); + rectangle(frame, boundingBox1, mt.colors[0], 2, 1); mt.addTarget(frame, boundingBox2, tracker_algorithm_name); - rectangle(image, boundingBox2, mt.colors[1], 2, 1); + rectangle(frame, boundingBox2, mt.colors[1], 2, 1); initialized = true; } else { //updates the tracker - if (mt.update_opt(frame)) + if (mt.update(frame)) { for (int i = 0; i < mt.targetNum; i++) rectangle(frame, mt.boundingBoxes[i], mt.colors[i], 2, 1); @@ -204,7 +200,7 @@ int main() imshow("Tracking API", frame); #ifdef RECORD_VIDEO_FLG - outputVideo << image; + outputVideo << frame; #endif diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0dd3aa4a8..d5fef17c9 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -19,7 +19,12 @@ namespace cv trackers.push_back(tracker); //Assign a random color to target - colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + if (targetNum == 1) + colors.push_back(Scalar(0, 0, 255)); + else + colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + + //Target counter targetNum++; @@ -75,17 +80,14 @@ namespace cv bool DETECT_FLG = false; - //printf("%d\n", targetNum); //Detect all for (int k = 0; k < targetNum; k++) tmpCandidates[k] = boundingBoxes[k]; - //if (ocl::haveOpenCL()) + if (ocl::haveOpenCL()) + ocl_detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); + else detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); - //else - //DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - //printf("BOOOLZZZ %d\n", detect_flgs[0]); - //printf("BOOOLXXX %d\n", detect_flgs[1]); for (int k = 0; k < targetNum; k++) { //TLD Tracker data extraction @@ -95,7 +97,6 @@ namespace cv tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); Ptr data = tracker->data; - /////// data->frameNum++; for (int i = 0; i < 2; i++) @@ -125,14 +126,9 @@ namespace cv trackerNeedsReInit[k] = false; } } - //printf("CanditateRes Size: %d \n", candidatesRes[k].size()); + std::vector::iterator it = std::max_element(candidatesRes[k].begin(), candidatesRes[k].end()); - //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); - //for( int i = 0; i < (int)candidatesRes.size(); i++ ) - //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); - //data->printme(); - //tldModel->printme(stdout); if (it == candidatesRes[k].end()) { @@ -445,9 +441,260 @@ namespace cv //printf("%d %f %f\n", k, srValue, scValue); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); + + + if (maxSc < 0) + detect_flgs[k] = false; + else + { + res[k] = maxScRect; + //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); + detect_flgs[k] = true; + } + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); + } + + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, + std::vector> &trackers) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Size initSize = tldModel->getMinSize(); + + for (int k = 0; k < trackers.size(); k++) + patches[k].clear(); + + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + Mat tmp; + int dx = initSize.width / 10, dy = initSize.height / 10; + Size2d size = img.size(); + double scale = 1.0; + int npos = 0, nneg = 0; + double maxSc = -5.0; + Rect2d maxScRect; + int scaleID; + std::vector resized_imgs, blurred_imgs; + + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + + std::vector tmpP; + std::vector tmpI; + + //int64 e1, e2; + //double t; + //e1 = getTickCount(); + + //Detection part + //Generate windows and filter by variance + scaleID = 0; + resized_imgs.push_back(img); + blurred_imgs.push_back(imgBlurred); + do + { + Mat_ intImgP, intImgP2; + tld::TLDDetector::computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2); + for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++) + { + for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++) + { + //Optimized variance calculation + int x = dx * i, + y = dy * j, + width = initSize.width, + height = initSize.height; + double p = 0, p2 = 0; + double A, B, C, D; + + A = intImgP(y, x); + B = intImgP(y, x + width); + C = intImgP(y + height, x); + D = intImgP(y + height, x + width); + p = (A + D - B - C) / (width * height); + + A = intImgP2(y, x); + B = intImgP2(y, x + width); + C = intImgP2(y + height, x); + D = intImgP2(y + height, x + width); + p2 = (A + D - B - C) / (width * height); + double windowVar = p2 - p * p; + + //Loop for on all objects + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + //Optimized variance calculation + bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); + + if (!varPass) + continue; + varBuffer[k].push_back(Point(dx * i, dy * j)); + varScaleIDs[k].push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); + } + } + } + scaleID++; + size.width /= tld::SCALE_STEP; + size.height /= tld::SCALE_STEP; + scale *= tld::SCALE_STEP; + resize(img, tmp, size, 0, 0, tld::DOWNSCALE_MODE); + resized_imgs.push_back(tmp); + GaussianBlur(resized_imgs[scaleID], tmp, tld::GaussBlurKernelSize, 0.0f); + blurred_imgs.push_back(tmp); + } while (size.width >= initSize.width && size.height >= initSize.height); + + + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Variance: %d\t%f\n", varBuffer.size(), t); + + //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); + + //Encsemble classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + + double ensRes = 0; + uchar* data = &blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x); + for (int x = 0; x < (int)tldModel->detector->classifiers.size(); x++) + { + int position = 0; + for (int n = 0; n < (int)tldModel->detector->classifiers[x].measurements.size(); n++) + { + position = position << 1; + if (data[tldModel->detector->classifiers[x].offset[n].x] < data[tldModel->detector->classifiers[x].offset[n].y]) + position++; + } + double posNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].x; + double negNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + continue; + else + ensRes += posNum / (posNum + negNum); + } + ensRes /= tldModel->detector->classifiers.size(); + ensRes = tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)); + + if (ensRes <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + /* + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + */ + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + + //printf("varBuffer 1: %d\n", varBuffer[0].size()); + //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); + + //printf("varBuffer 2: %d\n", varBuffer[1].size()); + //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); + + //NN classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + //Size InitSize = tldModel->getMinSize(); + npos = 0; + nneg = 0; + maxSc = -5.0; + + //Prepare batch of patches + int numOfPatches = (int)ensBuffer[k].size(); + Mat_ stdPatches(numOfPatches, 225); + double *resultSr = new double[numOfPatches]; + double *resultSc = new double[numOfPatches]; + + uchar *patchesData = stdPatches.data; + for (int i = 0; i < (int)ensBuffer.size(); i++) + { + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + uchar *stdPatchData = standardPatch.data; + for (int j = 0; j < 225; j++) + patchesData[225 * i + j] = stdPatchData[j]; + } + //Calculate Sr and Sc batches + tldModel->detector->ocl_batchSrSc(stdPatches, resultSr, resultSc, numOfPatches); + + for (int i = 0; i < (int)ensBuffer[k].size(); i++) + { + tld::TLDDetector::LabeledPatch labPatch; + standardPatch.data = &stdPatches.data[225 * i]; + double curScale = pow(tld::SCALE_STEP, ensScaleIDs[k][i]); + labPatch.rect = Rect2d(ensBuffer[k][i].x*curScale, ensBuffer[k][i].y*curScale, initSize.width * curScale, initSize.height * curScale); + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + + double srValue, scValue; + srValue = resultSr[i]; + + ////To fix: Check the paper, probably this cause wrong learning + // + labPatch.isObject = srValue > tld::THETA_NN; + labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1; + patches[k].push_back(labPatch); + // + + if (!labPatch.isObject) + { + nneg++; + continue; + } + else + { + npos++; + } + scValue = resultSc[i]; + if (scValue > maxSc) + { + maxSc = scValue; + maxScRect = labPatch.rect; + } + //printf("%d %f %f\n", k, srValue, scValue); + } + + if (maxSc < 0) detect_flgs[k] = false; @@ -458,5 +705,9 @@ namespace cv detect_flgs[k] = true; } } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); } + } \ No newline at end of file diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 2af2d8e8e..760cf9887 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -129,7 +129,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { - if (!ocl::haveOpenCL()) + if (ocl::haveOpenCL()) DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); else DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); From e8a23dbb229c3842238ef586f2555ed296a88db3 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 19:50:04 +0900 Subject: [PATCH 11/32] Fixed spaces in >> --- modules/tracking/src/multiTracker.cpp | 22 +++++++++++----------- modules/tracking/src/multiTracker.hpp | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index d5fef17c9..0e6133ce0 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -71,9 +71,9 @@ namespace cv //best overlap around 92% Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); - std::vector> detectorResults(targetNum); - std::vector> candidates(targetNum); - std::vector> candidatesRes(targetNum); + std::vector > detectorResults(targetNum); + std::vector > candidates(targetNum); + std::vector > candidatesRes(targetNum); std::vector tmpCandidates(targetNum); std::vector detect_flgs(targetNum); std::vector trackerNeedsReInit(targetNum); @@ -221,8 +221,8 @@ namespace cv return true; } - void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, - std::vector> &trackers) + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, + std::vector > &trackers) { //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; @@ -245,8 +245,8 @@ namespace cv int scaleID; std::vector resized_imgs, blurred_imgs; - std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); - std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); std::vector tmpP; std::vector tmpI; @@ -457,8 +457,8 @@ namespace cv //printf("NN: %d\t%f\n", patches.size(), t); } - void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, - std::vector> &trackers) + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, + std::vector > &trackers) { //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; @@ -481,8 +481,8 @@ namespace cv int scaleID; std::vector resized_imgs, blurred_imgs; - std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); - std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); std::vector tmpP; std::vector tmpI; diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index a2b0a01ea..9dc65c0e3 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -7,10 +7,10 @@ namespace cv { - void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, - std::vector& detect_flgs, std::vector>& trackers); - void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, - std::vector& detect_flgs, std::vector>& trackers); + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, + std::vector& detect_flgs, std::vector >& trackers); + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, + std::vector& detect_flgs, std::vector >& trackers); std::vector debugStack[10]; } #endif \ No newline at end of file From b42cad6cc1106e20c8de2c27df2e0cf9d852cce3 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 21:28:52 +0900 Subject: [PATCH 12/32] Removed inline specifiers --- modules/tracking/src/tldDetector.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 4dc433607..6e8878b01 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -73,8 +73,8 @@ namespace cv public: TLDDetector(){} ~TLDDetector(){} - inline double ensembleClassifierNum(const uchar* data); - inline void prepareClassifiers(int rowstep); + double ensembleClassifierNum(const uchar* data); + void prepareClassifiers(int rowstep); double Sr(const Mat_& patch); double ocl_Sr(const Mat_& patch); double Sc(const Mat_& patch); From 9ff31c4f6060c6e1f25b2121874803c2d18e3573 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 22:04:06 +0900 Subject: [PATCH 13/32] Added OpenCV lincense and "precomp.hpp" --- modules/tracking/src/multiTracker.cpp | 41 ++++++++++++++++++++++++++ modules/tracking/src/multiTracker.hpp | 42 +++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0e6133ce0..c3e314ea6 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "multiTracker.hpp" namespace cv diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index 9dc65c0e3..c98d7df66 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -1,6 +1,48 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #ifndef OPENCV_MULTITRACKER #define OPENCV_MULTITRACKER +#include "precomp.hpp" #include "tldTracker.hpp" #include "tldUtils.hpp" #include From cac774cbfc17863a40b950f39b63dbdbba7872a4 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 26 Jul 2015 23:18:52 +0900 Subject: [PATCH 14/32] Added Multi-tracker functionality and example 1. Multi-tracker classes (multiTracker.cpp) 2. Multi-tracker example (multiTracker_test.cpp) 3. Fixed a rare bug (OpenCL runtime error) --- .../include/opencv2/tracking/tracker.hpp | 24 ++ .../tracking/samples/multiTracker_test.cpp | 227 ++++++++++++++++++ modules/tracking/src/multiTracker.cpp | 49 ++++ modules/tracking/src/tldDetector.cpp | 2 +- 4 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 modules/tracking/samples/multiTracker_test.cpp create mode 100644 modules/tracking/src/multiTracker.cpp diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 09ef94f66..326daad86 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -49,6 +49,7 @@ #include "onlineBoosting.hpp" #include + #define BOILERPLATE_CODE(name,classname) \ static Ptr createTracker(const classname::Params ¶meters=classname::Params());\ virtual ~classname(){}; @@ -576,6 +577,7 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm Ptr model; }; + /************************************ Specific TrackerStateEstimator Classes ************************************/ /** @brief TrackerStateEstimator based on Boosting @@ -1243,6 +1245,28 @@ class CV_EXPORTS_W TrackerKCF : public Tracker BOILERPLATE_CODE("KCF",TrackerKCF); }; +/************************************ Multi-Tracker Classes ************************************/ + +class CV_EXPORTS_W MultiTracker +{ +public: + + bool addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name); + + bool update(const Mat& image); + + int targetNum = 0; + std::vector > trackers; + std::vector boundingBoxes; + std::vector colors; +}; + +class CV_EXPORTS_W MultiTrackerTLD : public MultiTracker +{ +public: + bool update(const Mat& image); +}; + //! @} } /* namespace cv */ diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp new file mode 100644 index 000000000..ac3c37382 --- /dev/null +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -0,0 +1,227 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +#define NUM_TEST_FRAMES 100 +#define TEST_VIDEO_INDEX 7 //TLD Dataset Video Index from 1-10 +//#define RECORD_VIDEO_FLG + +static Mat image; +static bool paused; +static bool selectObject = false; +static bool startSelection = false; +Rect2d boundingBox; + +static void onMouse(int event, int x, int y, int, void*) +{ + if (!selectObject) + { + switch (event) + { + case EVENT_LBUTTONDOWN: + //set origin of the bounding box + startSelection = true; + boundingBox.x = x; + boundingBox.y = y; + boundingBox.width = boundingBox.height = 0; + break; + case EVENT_LBUTTONUP: + //sei with and height of the bounding box + boundingBox.width = std::abs(x - boundingBox.x); + boundingBox.height = std::abs(y - boundingBox.y); + paused = false; + selectObject = true; + break; + case EVENT_MOUSEMOVE: + + if (startSelection && !selectObject) + { + //draw the bounding box + Mat currentFrame; + image.copyTo(currentFrame); + rectangle(currentFrame, Point((int)boundingBox.x, (int)boundingBox.y), Point(x, y), Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", currentFrame); + } + break; + } + } +} + +int main() +{ + // + // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" + // + char* tracker_algorithm_name = "TLD"; + + Mat frame; + paused = false; + namedWindow("Tracking API", 0); + setMouseCallback("Tracking API", onMouse, 0); + + MultiTrackerTLD mt; + + //Get the first frame + ////Open the capture + // VideoCapture cap(0); + // if( !cap.isOpened() ) + // { + // cout << "Video stream error"; + // return; + // } + //cap >> frame; + + //From TLD dataset + selectObject = true; + Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/TLD_dataset"); + Rect2d boundingBox2; + boundingBox2.x = 280; + boundingBox2.y = 60; + boundingBox2.width = 40; + boundingBox2.height = 60; + + frame = tld::tld_getNextDatasetFrame(); + frame.copyTo(image); + + // Setup output video +#ifdef RECORD_VIDEO_FLG + String outputFilename = "test.avi"; + VideoWriter outputVideo; + outputVideo.open(outputFilename, -1, 30, Size(image.cols, image.rows)); + + if (!outputVideo.isOpened()) + { + std::cout << "!!! Output video could not be opened" << std::endl; + getchar(); + return; + } +#endif + + rectangle(image, boundingBox, Scalar(255, 0, 0), 2, 1); + imshow("Tracking API", image); + + + bool initialized = false; + int frameCounter = 0; + + //Time measurment + int64 e3 = getTickCount(); + + for (;;) + { + //Time measurment + int64 e1 = getTickCount(); + //Frame num + frameCounter++; + if (frameCounter == NUM_TEST_FRAMES) break; + + char c = (char)waitKey(2); + if (c == 'q' || c == 27) + break; + if (c == 'p') + paused = !paused; + + if (!paused) + { + //cap >> frame; + frame = tld::tld_getNextDatasetFrame(); + if (frame.empty()) + { + break; + } + frame.copyTo(image); + + if (selectObject) + { + if (!initialized) + { + //initializes the tracker + mt.addTarget(frame, boundingBox1, tracker_algorithm_name); + rectangle(image, boundingBox1, mt.colors[0], 2, 1); + + + mt.addTarget(frame, boundingBox2, tracker_algorithm_name); + rectangle(image, boundingBox2, mt.colors[1], 2, 1); + initialized = true; + } + else + { + //updates the tracker + if (mt.update(frame)) + for (int i=0; i < mt.targetNum; i++) + rectangle(image, mt.boundingBoxes[i], mt.colors[i], 2, 1); + } + } + imshow("Tracking API", image); + +#ifdef RECORD_VIDEO_FLG + outputVideo << image; +#endif + + + //Time measurment + int64 e2 = getTickCount(); + double t1 = (e2 - e1) / getTickFrequency(); + cout << frameCounter << "\tframe : " << t1 * 1000.0 << "ms" << endl; + + waitKey(0); + } + } + + //Time measurment + int64 e4 = getTickCount(); + double t2 = (e4 - e3) / getTickFrequency(); + cout << "Average Time for Frame: " << t2 * 1000.0 / frameCounter << "ms" << endl; + cout << "Average FPS: " << 1.0 / t2*frameCounter << endl; + + + waitKey(0); + + return 0; +} \ No newline at end of file diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp new file mode 100644 index 000000000..0ce6e55ca --- /dev/null +++ b/modules/tracking/src/multiTracker.cpp @@ -0,0 +1,49 @@ +#include + +namespace cv +{ + //Multitracker + bool MultiTracker::addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name) + { + Ptr tracker = Tracker::create(tracker_algorithm_name); + if (tracker == NULL) + return false; + + if (!tracker->init(image, boundingBox)) + return false; + + //Add BB of target + boundingBoxes.push_back(boundingBox); + + //Add Tracker to stack + trackers.push_back(tracker); + + //Assign a random color to target + colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + + //Target counter + targetNum++; + + return true; + } + + bool MultiTracker::update(const Mat& image) + { + for (int i = 0; i < trackers.size(); i++) + if (!trackers[i]->update(image, boundingBoxes[i])) + return false; + + return true; + } + + //Multitracker TLD + /*Optimized update method for TLD Multitracker */ + bool MultiTrackerTLD::update(const Mat& image) + { + for (int i = 0; i < trackers.size(); i++) + if (!trackers[i]->update(image, boundingBoxes[i])) + return false; + + return true; + } +} \ No newline at end of file diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index cbb783ece..900761bef 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -219,7 +219,7 @@ namespace cv // 2 -> Pos&Neg size_t globSize = 2 * numOfPatches*MAX_EXAMPLES_IN_MODEL; //e3 = getTickCount(); - if (!k.run(1, &globSize, NULL, false)) + if (!k.run(1, &globSize, NULL, true)) printf("Kernel Run Error!!!"); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; From 871f40cb5b40c1e75972e0bae37f9a1a285b310b Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 27 Jul 2015 15:41:19 +0900 Subject: [PATCH 15/32] Added optimization to Multi-target TLD update --- .../include/opencv2/tracking/tracker.hpp | 2 +- modules/tracking/src/multiTracker.cpp | 150 +++++++++++++++++- modules/tracking/src/tldTracker.hpp | 1 - 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 326daad86..a9d104a0b 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -565,7 +565,7 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm virtual void read( const FileNode& fn )=0; virtual void write( FileStorage& fs ) const=0; - protected: + public: virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0ce6e55ca..92c7f51b1 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,4 +1,4 @@ -#include +#include "tldTracker.hpp" namespace cv { @@ -35,14 +35,156 @@ namespace cv return true; } - + //Multitracker TLD /*Optimized update method for TLD Multitracker */ bool MultiTrackerTLD::update(const Mat& image) { - for (int i = 0; i < trackers.size(); i++) - if (!trackers[i]->update(image, boundingBoxes[i])) + + for (int k = 0; k < trackers.size(); k++) + { + //Set current target(tracker) parameters + Rect2d boundingBox = boundingBoxes[k]; + Ptr tracker = (Ptr)static_cast> (trackers[k]); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + double scale = data->getScale(); + + + Mat image_gray, image_blurred, imageForDetector; + cvtColor(image, image_gray, COLOR_BGR2GRAY); + + if (scale > 1.0) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + + data->frameNum++; + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + std::vector detectorResults; + //best overlap around 92% + + std::vector candidates; + std::vector candidatesRes; + bool trackerNeedsReInit = false; + bool DETECT_FLG = false; + for (int i = 0; i < 2; i++) + { + Rect2d tmpCandid = boundingBox; + + if (i == 1) + { + if (ocl::haveOpenCL()) + DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + else + DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + } + + if (((i == 0) && !data->failedLastTime && tracker->trackerProxy->update(image, tmpCandid)) || (DETECT_FLG)) + { + candidates.push_back(tmpCandid); + if (i == 0) + tld::resample(image_gray, tmpCandid, standardPatch); + else + tld::resample(imageForDetector, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + } + else + { + if (i == 0) + trackerNeedsReInit = true; + } + } + + std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + + //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); + //for( int i = 0; i < (int)candidatesRes.size(); i++ ) + //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); + //data->printme(); + //tldModel->printme(stdout); + + if (it == candidatesRes.end()) + { + data->confident = false; + data->failedLastTime = true; return false; + } + else + { + boundingBox = candidates[it - candidatesRes.begin()]; + boundingBoxes[k] = boundingBox; + data->failedLastTime = false; + if (trackerNeedsReInit || it != candidatesRes.begin()) + tracker->trackerProxy->init(image, boundingBox); + } + +#if 1 + if (it != candidatesRes.end()) + { + tld::resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); + //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) + //dfprintf((stderr, "detector WON\n")); + } + else + { + //dfprintf((stderr, "%d x x\n", data->frameNum)); + } +#endif + + if (*it > tld::CORE_THRESHOLD) + data->confident = true; + + if (data->confident) + { + tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, tracker->params, data->getMinSize()); + tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, tracker->params); + std::vector > examplesForModel, examplesForEnsemble; + examplesForModel.reserve(100); examplesForEnsemble.reserve(100); + int negRelabeled = 0; + for (int i = 0; i < (int)detectorResults.size(); i++) + { + bool expertResult; + if (detectorResults[i].isObject) + { + expertResult = nExpert(detectorResults[i].rect); + if (expertResult != detectorResults[i].isObject) + negRelabeled++; + } + else + { + expertResult = pExpert(detectorResults[i].rect); + } + + detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); + detectorResults[i].isObject = expertResult; + } + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + //dprintf(("%d relabeled by nExpert\n", negRelabeled)); + pExpert.additionalExamples(examplesForModel, examplesForEnsemble); + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); + examplesForModel.clear(); examplesForEnsemble.clear(); + nExpert.additionalExamples(examplesForModel, examplesForEnsemble); + + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); + } + else + { +#ifdef CLOSED_LOOP + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); +#endif + } + + + } return true; } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index e04926d5d..40de754b7 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -128,7 +128,6 @@ public: void read(const FileNode& fn); void write(FileStorage& fs) const; -protected: class Pexpert { public: From ada268148de0987c26125cf7dc2bbd03a3489d32 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:44:14 +0900 Subject: [PATCH 16/32] Added VOT 2015 dataset support http://www.votchallenge.net/vot2015/dataset.html --- .../include/opencv2/tracking/tldDataset.hpp | 2 +- modules/tracking/src/tldDataset.cpp | 157 +++++++++++------- 2 files changed, 98 insertions(+), 61 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tldDataset.hpp b/modules/tracking/include/opencv2/tracking/tldDataset.hpp index 6d21a7d48..d31919ab5 100644 --- a/modules/tracking/include/opencv2/tracking/tldDataset.hpp +++ b/modules/tracking/include/opencv2/tracking/tldDataset.hpp @@ -48,7 +48,7 @@ namespace cv { namespace tld { - CV_EXPORTS cv::Rect2d tld_InitDataset(int datasetInd, const char* rootPath = "TLD_dataset"); + CV_EXPORTS cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath = "TLD_dataset", int datasetInd = 0); CV_EXPORTS cv::Mat tld_getNextDatasetFrame(); } } diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index b93de2576..ddb241a83 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -48,70 +48,105 @@ namespace cv char tldRootPath[100]; int frameNum = 0; bool flagPNG = false; + bool flagVOT = false; - cv::Rect2d tld_InitDataset(int datasetInd,const char* rootPath) + //TLD Dataset Parameters + char* tldFolderName[10] = { + "01_david", + "02_jumping", + "03_pedestrian1", + "04_pedestrian2", + "05_pedestrian3", + "06_car", + "07_motocross", + "08_volkswagen", + "09_carchase", + "10_panda" + }; + char* votFolderName[60] = { + "bag", "ball1", "ball2", "basketball", "birds1", "birds2", "blanket", "bmx", "bolt1", "bolt2", + "book", "butterfly", "car1", "car2", "crossing", "dinosaur", "fernando", "fish1", "fish2", "fish3", + "fish4", "girl", "glove", "godfather", "graduate", "gymnastics1", "gymnastics2 ", "gymnastics3", "gymnastics4", "hand", + "handball1", "handball2", "helicopter", "iceskater1", "iceskater2", "leaves", "marching", "matrix", "motocross1", "motocross2", + "nature", "octopus", "pedestrian1", "pedestrian2", "rabbit", "racing", "road", "shaking", "sheep", "singer1", + "singer2", "singer3", "soccer1", "soccer2", "soldier", "sphere", "tiger", "traffic", "tunnel", "wiper" + }; + + Rect2d tldInitBB[10] = { + Rect2d(165, 183, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(337, 219, 54, 37), Rect2d(58, 100, 27, 22) + }; + Rect2d votInitBB[60] = { + Rect2d(142, 125, 90, 39), Rect2d(490, 400, 40, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 100), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), + }; + + int tldFrameOffset[10] = { 100, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + int votFrameOffset[60] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + bool tldFlagPNG[10] = { 0, 0, 0, 0, 0, 0, 1, 0, 0, 0 }; + bool votFlagPNG[60] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath, int datasetInd) { char* folderName = (char *)""; - int x = 0; - int y = 0; - int w = 0; - int h = 0; - flagPNG = false; - - frameNum = 1; - - if (datasetInd == 1) { - folderName = (char *)"01_david"; - x = 165, y = 83; - w = 51; h = 54; - frameNum = 100; - } - if (datasetInd == 2) { - folderName = (char *)"02_jumping"; - x = 147, y = 110; - w = 33; h = 32; - } - if (datasetInd == 3) { - folderName = (char *)"03_pedestrian1"; - x = 47, y = 51; - w = 21; h = 36; - } - if (datasetInd == 4) { - folderName = (char *)"04_pedestrian2"; - x = 130, y = 134; - w = 21; h = 53; - } - if (datasetInd == 5) { - folderName = (char *)"05_pedestrian3"; - x = 154, y = 102; - w = 24; h = 52; - } - if (datasetInd == 6) { - folderName = (char *)"06_car"; - x = 142, y = 125; - w = 90; h = 39; - } - if (datasetInd == 7) { - folderName = (char *)"07_motocross"; - x = 290, y = 43; - w = 23; h = 40; - flagPNG = true; - } - if (datasetInd == 8) { - folderName = (char *)"08_volkswagen"; - x = 273, y = 77; - w = 27; h = 25; - } - if (datasetInd == 9) { - folderName = (char *)"09_carchase"; - x = 145, y = 84; - w = 54; h = 37; - } - if (datasetInd == 10){ - folderName = (char *)"10_panda"; - x = 58, y = 100; - w = 27; h = 22; + int x, y, w, h; + + //Index range + // 1-10 TLD Dataset + // 1-60 VOT 2015 Dataset + int id = videoInd - 1; + + if (datasetInd == 0) + { + folderName = tldFolderName[id]; + x = tldInitBB[id].x; + y = tldInitBB[id].y; + w = tldInitBB[id].width; + h = tldInitBB[id].height; + frameNum = tldFrameOffset[id]; + flagPNG = tldFlagPNG[id]; + flagVOT = false; } + if (datasetInd == 1) + { + folderName = votFolderName[id]; + x = votInitBB[id].x; + y = votInitBB[id].y; + w = votInitBB[id].width; + h = votInitBB[id].height; + frameNum = votFrameOffset[id]; + flagPNG = votFlagPNG[id]; + flagVOT = true; + } strcpy(tldRootPath, rootPath); strcat(tldRootPath, "\\"); @@ -127,6 +162,8 @@ namespace cv char numStr[10]; strcpy(fullPath, tldRootPath); strcat(fullPath, "\\"); + if (flagVOT) + strcat(fullPath, "000"); if (frameNum < 10) strcat(fullPath, "0000"); else if (frameNum < 100) strcat(fullPath, "000"); else if (frameNum < 1000) strcat(fullPath, "00"); From b318e38bd0058382b2db633598de7b0105788be9 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:46:33 +0900 Subject: [PATCH 17/32] Debug through candidates display --- modules/tracking/src/tldDetector.cpp | 14 ++++++++++++-- modules/tracking/src/tldDetector.hpp | 14 ++++++++------ modules/tracking/src/tldTracker.cpp | 22 ++++++++++++++++++---- modules/tracking/src/tldTracker.hpp | 6 ------ 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 900761bef..0443f46cb 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -491,6 +491,10 @@ namespace cv continue; varBuffer.push_back(Point(dx * i, dy * j)); varScaleIDs.push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, initSize.width*curScale, initSize.height*curScale)); } } scaleID++; @@ -520,6 +524,9 @@ namespace cv //t = (e2 - e1) / getTickFrequency()*1000.0; //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); + //printf("varBuffer: %d\n", varBuffer.size()); + //printf("ensBuffer: %d\n", ensBuffer.size()); + //NN classification //e1 = getTickCount(); for (int i = 0; i < (int)ensBuffer.size(); i++) @@ -561,8 +568,11 @@ namespace cv if (maxSc < 0) return false; - res = maxScRect; - return true; + else + { + res = maxScRect; + return true; + } } bool TLDDetector::ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize) diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 3e1189592..4dc433607 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -66,6 +66,8 @@ namespace cv static const cv::Size GaussBlurKernelSize(3, 3); + + class TLDDetector { public: @@ -79,6 +81,7 @@ namespace cv double ocl_Sc(const Mat_& patch); void ocl_batchSrSc(const Mat_& patches, double *resultSr, double *resultSc, int numOfPatches); + std::vector debugStack[10]; std::vector classifiers; Mat *posExp, *negExp; int *posNum, *negNum; @@ -93,15 +96,14 @@ namespace cv bool isObject, shouldBeIntegrated; }; bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); - bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); - protected: - - + bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector& patches, Size initSize); friend class MyMouseCallbackDEBUG; - void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } - inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); + static void computeIntegralImages(const Mat& img, Mat_& intImgP, Mat_& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); } + static inline bool patchVariance(Mat_& intImgP, Mat_& intImgP2, double *originalVariance, Point pt, Size size); }; + + } } diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 0f1e2f78e..2af2d8e8e 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -45,6 +45,13 @@ namespace cv { + TrackerTLD::Params::Params(){} + + void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} + + void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} + + Ptr TrackerTLD::createTracker(const TrackerTLD::Params ¶meters) { return Ptr(new tld::TrackerTLDImpl(parameters)); @@ -112,7 +119,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) Mat_ standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); std::vector detectorResults; //best overlap around 92% - std::vector candidates; std::vector candidatesRes; bool trackerNeedsReInit = false; @@ -123,12 +129,11 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { - if (ocl::haveOpenCL()) + if (!ocl::haveOpenCL()) DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); else DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); } - if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG)) { candidates.push_back(tmpCandid); @@ -144,7 +149,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) trackerNeedsReInit = true; } } - std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); @@ -230,6 +234,16 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) #endif } + + //Debug display candidates after Variance Filter + //////////////////////////////////////////////// + Mat tmpImg = image; + for (int i = 0; i < tldModel->detector->debugStack[0].size(); i++) + //rectangle(tmpImg, tldModel->detector->debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); + tldModel->detector->debugStack[0].clear(); + tmpImg.copyTo(image); + + //////////////////////////////////////////////// return true; } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index 40de754b7..4a5610de3 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -52,12 +52,6 @@ namespace cv { -TrackerTLD::Params::Params(){} - -void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){} - -void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {} - namespace tld { class TrackerProxy From 2088e5e6f9dc3322e6467e662b7b693453f221b0 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 9 Aug 2015 04:49:32 +0900 Subject: [PATCH 18/32] Improved VF optimization + Added EC optimization for MO-TLD --- .../include/opencv2/tracking/tracker.hpp | 2 +- .../tracking/samples/multiTracker_test.cpp | 25 +- modules/tracking/src/multiTracker.cpp | 385 +++++++++++++++--- .../tracking/src/tldEnsembleClassifier.hpp | 2 +- 4 files changed, 344 insertions(+), 70 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index a9d104a0b..54adbe301 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1264,7 +1264,7 @@ public: class CV_EXPORTS_W MultiTrackerTLD : public MultiTracker { public: - bool update(const Mat& image); + bool update_opt(const Mat& image); }; //! @} diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index ac3c37382..6831c1f88 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -49,7 +49,7 @@ using namespace std; using namespace cv; #define NUM_TEST_FRAMES 100 -#define TEST_VIDEO_INDEX 7 //TLD Dataset Video Index from 1-10 +#define TEST_VIDEO_INDEX 15 //TLD Dataset Video Index from 1-10 for TLD and 1-60 for VOT //#define RECORD_VIDEO_FLG static Mat image; @@ -119,12 +119,12 @@ int main() //From TLD dataset selectObject = true; - Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/TLD_dataset"); + Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/VOT 2015", 1); Rect2d boundingBox2; - boundingBox2.x = 280; - boundingBox2.y = 60; - boundingBox2.width = 40; - boundingBox2.height = 60; + boundingBox2.x = 470; + boundingBox2.y = 500; + boundingBox2.width = 50; + boundingBox2.height = 100; frame = tld::tld_getNextDatasetFrame(); frame.copyTo(image); @@ -140,6 +140,7 @@ int main() std::cout << "!!! Output video could not be opened" << std::endl; getchar(); return; + } #endif @@ -193,12 +194,14 @@ int main() else { //updates the tracker - if (mt.update(frame)) - for (int i=0; i < mt.targetNum; i++) - rectangle(image, mt.boundingBoxes[i], mt.colors[i], 2, 1); + if (mt.update_opt(frame)) + { + for (int i = 0; i < mt.targetNum; i++) + rectangle(frame, mt.boundingBoxes[i], mt.colors[i], 2, 1); + } } } - imshow("Tracking API", image); + imshow("Tracking API", frame); #ifdef RECORD_VIDEO_FLG outputVideo << image; @@ -210,7 +213,7 @@ int main() double t1 = (e2 - e1) / getTickFrequency(); cout << frameCounter << "\tframe : " << t1 * 1000.0 << "ms" << endl; - waitKey(0); + //waitKey(0); } } diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 92c7f51b1..0dd3aa4a8 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,4 +1,4 @@ -#include "tldTracker.hpp" +#include "multiTracker.hpp" namespace cv { @@ -29,75 +29,104 @@ namespace cv bool MultiTracker::update(const Mat& image) { + printf("Naive-Loop MO-TLD Update....\n"); for (int i = 0; i < trackers.size(); i++) if (!trackers[i]->update(image, boundingBoxes[i])) return false; return true; } - + //Multitracker TLD /*Optimized update method for TLD Multitracker */ - bool MultiTrackerTLD::update(const Mat& image) + bool MultiTrackerTLD::update_opt(const Mat& image) { - - for (int k = 0; k < trackers.size(); k++) - { - //Set current target(tracker) parameters - Rect2d boundingBox = boundingBoxes[k]; - Ptr tracker = (Ptr)static_cast> (trackers[k]); - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); - Ptr data = tracker->data; - double scale = data->getScale(); + printf("Optimized MO-TLD Update....\n"); + //Get parameters from first object + //Set current target(tracker) parameters + Rect2d boundingBox = boundingBoxes[0]; + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + double scale = data->getScale(); - Mat image_gray, image_blurred, imageForDetector; - cvtColor(image, image_gray, COLOR_BGR2GRAY); + Mat image_gray, image_blurred, imageForDetector; + cvtColor(image, image_gray, COLOR_BGR2GRAY); - if (scale > 1.0) - resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); - else - imageForDetector = image_gray; - GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + if (scale > 1.0) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + + //best overlap around 92% + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + + std::vector> detectorResults(targetNum); + std::vector> candidates(targetNum); + std::vector> candidatesRes(targetNum); + std::vector tmpCandidates(targetNum); + std::vector detect_flgs(targetNum); + std::vector trackerNeedsReInit(targetNum); + + bool DETECT_FLG = false; + + //printf("%d\n", targetNum); + //Detect all + for (int k = 0; k < targetNum; k++) + tmpCandidates[k] = boundingBoxes[k]; + //if (ocl::haveOpenCL()) + detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); + //else + //DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + + //printf("BOOOLZZZ %d\n", detect_flgs[0]); + //printf("BOOOLXXX %d\n", detect_flgs[1]); + for (int k = 0; k < targetNum; k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + /////// data->frameNum++; - Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); - std::vector detectorResults; - //best overlap around 92% - - std::vector candidates; - std::vector candidatesRes; - bool trackerNeedsReInit = false; - bool DETECT_FLG = false; + for (int i = 0; i < 2; i++) { - Rect2d tmpCandid = boundingBox; + Rect2d tmpCandid = boundingBoxes[k]; - if (i == 1) + //if (i == 1) { - if (ocl::haveOpenCL()) - DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - else - DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + DETECT_FLG = detect_flgs[k]; + tmpCandid = tmpCandidates[k]; } if (((i == 0) && !data->failedLastTime && tracker->trackerProxy->update(image, tmpCandid)) || (DETECT_FLG)) { - candidates.push_back(tmpCandid); + candidates[k].push_back(tmpCandid); if (i == 0) tld::resample(image_gray, tmpCandid, standardPatch); else tld::resample(imageForDetector, tmpCandid, standardPatch); - candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + candidatesRes[k].push_back(tldModel->detector->Sc(standardPatch)); } else { if (i == 0) - trackerNeedsReInit = true; + trackerNeedsReInit[k] = true; + else + trackerNeedsReInit[k] = false; } } - - std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + //printf("CanditateRes Size: %d \n", candidatesRes[k].size()); + std::vector::iterator it = std::max_element(candidatesRes[k].begin(), candidatesRes[k].end()); //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); //for( int i = 0; i < (int)candidatesRes.size(); i++ ) @@ -105,25 +134,25 @@ namespace cv //data->printme(); //tldModel->printme(stdout); - if (it == candidatesRes.end()) + if (it == candidatesRes[k].end()) { + data->confident = false; data->failedLastTime = true; return false; } else { - boundingBox = candidates[it - candidatesRes.begin()]; - boundingBoxes[k] = boundingBox; + boundingBoxes[k] = candidates[k][it - candidatesRes[k].begin()]; data->failedLastTime = false; - if (trackerNeedsReInit || it != candidatesRes.begin()) - tracker->trackerProxy->init(image, boundingBox); + if (trackerNeedsReInit[k] || it != candidatesRes[k].begin()) + tracker->trackerProxy->init(image, boundingBoxes[k]); } #if 1 - if (it != candidatesRes.end()) + if (it != candidatesRes[k].end()) { - tld::resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + tld::resample(imageForDetector, candidates[k][it - candidatesRes[k].begin()], standardPatch); //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) //dfprintf((stderr, "detector WON\n")); @@ -139,29 +168,29 @@ namespace cv if (data->confident) { - tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, tracker->params, data->getMinSize()); - tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, tracker->params); + tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBoxes[k], tldModel->detector, tracker->params, data->getMinSize()); + tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBoxes[k], tldModel->detector, tracker->params); std::vector > examplesForModel, examplesForEnsemble; examplesForModel.reserve(100); examplesForEnsemble.reserve(100); int negRelabeled = 0; - for (int i = 0; i < (int)detectorResults.size(); i++) + for (int i = 0; i < (int)detectorResults[k].size(); i++) { bool expertResult; - if (detectorResults[i].isObject) + if (detectorResults[k][i].isObject) { - expertResult = nExpert(detectorResults[i].rect); - if (expertResult != detectorResults[i].isObject) + expertResult = nExpert(detectorResults[k][i].rect); + if (expertResult != detectorResults[k][i].isObject) negRelabeled++; } else { - expertResult = pExpert(detectorResults[i].rect); + expertResult = pExpert(detectorResults[k][i].rect); } - detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); - detectorResults[i].isObject = expertResult; + detectorResults[k][i].shouldBeIntegrated = detectorResults[k][i].shouldBeIntegrated || (detectorResults[k][i].isObject != expertResult); + detectorResults[k][i].isObject = expertResult; } - tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults[k]); //dprintf(("%d relabeled by nExpert\n", negRelabeled)); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); if (ocl::haveOpenCL()) @@ -183,9 +212,251 @@ namespace cv #endif } - - } + } + //Debug display candidates after Variance Filter + //////////////////////////////////////////////// + Mat tmpImg = image; + for (int i = 0; i < debugStack[0].size(); i++) + //rectangle(tmpImg, debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); + debugStack[0].clear(); + tmpImg.copyTo(image); + //////////////////////////////////////////////// return true; } + + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, + std::vector> &trackers) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Size initSize = tldModel->getMinSize(); + + for (int k = 0; k < trackers.size(); k++) + patches[k].clear(); + + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + Mat tmp; + int dx = initSize.width / 10, dy = initSize.height / 10; + Size2d size = img.size(); + double scale = 1.0; + int npos = 0, nneg = 0; + double maxSc = -5.0; + Rect2d maxScRect; + int scaleID; + std::vector resized_imgs, blurred_imgs; + + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + + std::vector tmpP; + std::vector tmpI; + + //int64 e1, e2; + //double t; + //e1 = getTickCount(); + + //Detection part + //Generate windows and filter by variance + scaleID = 0; + resized_imgs.push_back(img); + blurred_imgs.push_back(imgBlurred); + do + { + Mat_ intImgP, intImgP2; + tld::TLDDetector::computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2); + for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++) + { + for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++) + { + //Optimized variance calculation + int x = dx * i, + y = dy * j, + width = initSize.width, + height = initSize.height; + double p = 0, p2 = 0; + double A, B, C, D; + + A = intImgP(y, x); + B = intImgP(y, x + width); + C = intImgP(y + height, x); + D = intImgP(y + height, x + width); + p = (A + D - B - C) / (width * height); + + A = intImgP2(y, x); + B = intImgP2(y, x + width); + C = intImgP2(y + height, x); + D = intImgP2(y + height, x + width); + p2 = (A + D - B - C) / (width * height); + double windowVar = p2 - p * p; + + //Loop for on all objects + for (int k=0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + //Optimized variance calculation + bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); + + if (!varPass) + continue; + varBuffer[k].push_back(Point(dx * i, dy * j)); + varScaleIDs[k].push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); + } + } + } + scaleID++; + size.width /= tld::SCALE_STEP; + size.height /= tld::SCALE_STEP; + scale *= tld::SCALE_STEP; + resize(img, tmp, size, 0, 0, tld::DOWNSCALE_MODE); + resized_imgs.push_back(tmp); + GaussianBlur(resized_imgs[scaleID], tmp, tld::GaussBlurKernelSize, 0.0f); + blurred_imgs.push_back(tmp); + } while (size.width >= initSize.width && size.height >= initSize.height); + + + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Variance: %d\t%f\n", varBuffer.size(), t); + + //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); + + //Encsemble classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + + double ensRes = 0; + uchar* data = &blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x); + for (int x = 0; x < (int)tldModel->detector->classifiers.size(); x++) + { + int position = 0; + for (int n = 0; n < (int)tldModel->detector->classifiers[x].measurements.size(); n++) + { + position = position << 1; + if (data[tldModel->detector->classifiers[x].offset[n].x] < data[tldModel->detector->classifiers[x].offset[n].y]) + position++; + } + double posNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].x; + double negNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + continue; + else + ensRes += posNum / (posNum + negNum); + } + ensRes /= tldModel->detector->classifiers.size(); + ensRes = tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)); + + if ( ensRes <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + /* + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + */ + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); + + //printf("varBuffer 1: %d\n", varBuffer[0].size()); + //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); + + //printf("varBuffer 2: %d\n", varBuffer[1].size()); + //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); + + //NN classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + npos = 0; + nneg = 0; + maxSc = -5.0; + + for (int i = 0; i < (int)ensBuffer[k].size(); i++) + { + tld::TLDDetector::LabeledPatch labPatch; + double curScale = pow(tld::SCALE_STEP, ensScaleIDs[k][i]); + labPatch.rect = Rect2d(ensBuffer[k][i].x*curScale, ensBuffer[k][i].y*curScale, initSize.width * curScale, initSize.height * curScale); + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + + double srValue, scValue; + srValue = tldModel->detector->Sr(standardPatch); + + ////To fix: Check the paper, probably this cause wrong learning + // + labPatch.isObject = srValue > tld::THETA_NN; + labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1; + patches[k].push_back(labPatch); + // + + if (!labPatch.isObject) + { + nneg++; + continue; + } + else + { + npos++; + } + scValue = tldModel->detector->Sc(standardPatch); + if (scValue > maxSc) + { + maxSc = scValue; + maxScRect = labPatch.rect; + } + //printf("%d %f %f\n", k, srValue, scValue); + } + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); + + if (maxSc < 0) + detect_flgs[k] = false; + else + { + res[k] = maxScRect; + //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); + detect_flgs[k] = true; + } + } + } } \ No newline at end of file diff --git a/modules/tracking/src/tldEnsembleClassifier.hpp b/modules/tracking/src/tldEnsembleClassifier.hpp index 97221372d..f0ec175ba 100644 --- a/modules/tracking/src/tldEnsembleClassifier.hpp +++ b/modules/tracking/src/tldEnsembleClassifier.hpp @@ -54,7 +54,7 @@ namespace cv double posteriorProbability(const uchar* data, int rowstep) const; double posteriorProbabilityFast(const uchar* data) const; void prepareClassifier(int rowstep); - private: + TLDEnsembleClassifier(const std::vector& meas, int beg, int end); static void stepPrefSuff(std::vector & arr, int pos, int len, int gridSize); int code(const uchar* data, int rowstep) const; From 96ce5993fb06374f6117130d498d656be2ccb4d4 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 09:24:56 +0900 Subject: [PATCH 19/32] Added multiTracker.hpp (fix) --- modules/tracking/src/multiTracker.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 modules/tracking/src/multiTracker.hpp diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp new file mode 100644 index 000000000..a2b0a01ea --- /dev/null +++ b/modules/tracking/src/multiTracker.hpp @@ -0,0 +1,16 @@ +#ifndef OPENCV_MULTITRACKER +#define OPENCV_MULTITRACKER + +#include "tldTracker.hpp" +#include "tldUtils.hpp" +#include + +namespace cv +{ + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, + std::vector& detect_flgs, std::vector>& trackers); + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, + std::vector& detect_flgs, std::vector>& trackers); + std::vector debugStack[10]; +} +#endif \ No newline at end of file From 6a1bf675cade3a9357adfb3930052966bd911f19 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:36:21 +0900 Subject: [PATCH 20/32] Fix std::vector > --- modules/tracking/include/opencv2/tracking/tracker.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 54adbe301..8144e72bd 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1256,7 +1256,7 @@ public: bool update(const Mat& image); int targetNum = 0; - std::vector > trackers; + std::vector > trackers; std::vector boundingBoxes; std::vector colors; }; From 0e14ca781640d797e268b9b8ff161945e3af74cf Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:38:31 +0900 Subject: [PATCH 21/32] Fixed some init BB --- modules/tracking/src/tldDataset.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index ddb241a83..16d5596f2 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -73,14 +73,14 @@ namespace cv }; Rect2d tldInitBB[10] = { - Rect2d(165, 183, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), + Rect2d(165, 93, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(337, 219, 54, 37), Rect2d(58, 100, 27, 22) }; Rect2d votInitBB[60] = { Rect2d(142, 125, 90, 39), Rect2d(490, 400, 40, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), - Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), + Rect2d(450, 380, 60, 60), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), - Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 100), + Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(560, 460, 50, 120), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), From ed1c452dcf56eb2222b2759bda08bd7a62685e27 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:43:31 +0900 Subject: [PATCH 22/32] Fixed pointers to posNum and negNum --- modules/tracking/src/tldDetector.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index 0443f46cb..bfef69a45 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -131,8 +131,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devPositiveSamples), ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devNCC), - posNum, - negNum); + *posNum, + *negNum); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; @@ -208,8 +208,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devPosNCC), ocl::KernelArg::PtrWriteOnly(devNegNCC), - posNum, - negNum, + *posNum, + *negNum, numOfPatches); //e4 = getTickCount(); @@ -364,8 +364,8 @@ namespace cv ocl::KernelArg::PtrReadOnly(devPositiveSamples), ocl::KernelArg::PtrReadOnly(devNegativeSamples), ocl::KernelArg::PtrWriteOnly(devNCC), - posNum, - negNum); + *posNum, + *negNum); //e4 = getTickCount(); //t = (e4 - e3) / getTickFrequency()*1000.0; From d310f82e410ec61ed35a39c41ebb03b45fd670aa Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 17:45:21 +0900 Subject: [PATCH 23/32] Added MO-TLD NN Classifier optimization on GPU --- .../tracking/samples/multiTracker_test.cpp | 18 +- modules/tracking/src/multiTracker.cpp | 285 ++++++++++++++++-- modules/tracking/src/tldTracker.cpp | 2 +- 3 files changed, 276 insertions(+), 29 deletions(-) diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index 6831c1f88..c202a161f 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -120,11 +120,7 @@ int main() //From TLD dataset selectObject = true; Rect2d boundingBox1 = tld::tld_InitDataset(TEST_VIDEO_INDEX, "D:/opencv/VOT 2015", 1); - Rect2d boundingBox2; - boundingBox2.x = 470; - boundingBox2.y = 500; - boundingBox2.width = 50; - boundingBox2.height = 100; + Rect2d boundingBox2(470, 490, 50, 120); frame = tld::tld_getNextDatasetFrame(); frame.copyTo(image); @@ -133,13 +129,13 @@ int main() #ifdef RECORD_VIDEO_FLG String outputFilename = "test.avi"; VideoWriter outputVideo; - outputVideo.open(outputFilename, -1, 30, Size(image.cols, image.rows)); + outputVideo.open(outputFilename, -1, 15, Size(image.cols, image.rows)); if (!outputVideo.isOpened()) { std::cout << "!!! Output video could not be opened" << std::endl; getchar(); - return; + return 0; } #endif @@ -184,17 +180,17 @@ int main() { //initializes the tracker mt.addTarget(frame, boundingBox1, tracker_algorithm_name); - rectangle(image, boundingBox1, mt.colors[0], 2, 1); + rectangle(frame, boundingBox1, mt.colors[0], 2, 1); mt.addTarget(frame, boundingBox2, tracker_algorithm_name); - rectangle(image, boundingBox2, mt.colors[1], 2, 1); + rectangle(frame, boundingBox2, mt.colors[1], 2, 1); initialized = true; } else { //updates the tracker - if (mt.update_opt(frame)) + if (mt.update(frame)) { for (int i = 0; i < mt.targetNum; i++) rectangle(frame, mt.boundingBoxes[i], mt.colors[i], 2, 1); @@ -204,7 +200,7 @@ int main() imshow("Tracking API", frame); #ifdef RECORD_VIDEO_FLG - outputVideo << image; + outputVideo << frame; #endif diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0dd3aa4a8..d5fef17c9 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -19,7 +19,12 @@ namespace cv trackers.push_back(tracker); //Assign a random color to target - colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + if (targetNum == 1) + colors.push_back(Scalar(0, 0, 255)); + else + colors.push_back(Scalar(rand() % 256, rand() % 256, rand() % 256)); + + //Target counter targetNum++; @@ -75,17 +80,14 @@ namespace cv bool DETECT_FLG = false; - //printf("%d\n", targetNum); //Detect all for (int k = 0; k < targetNum; k++) tmpCandidates[k] = boundingBoxes[k]; - //if (ocl::haveOpenCL()) + if (ocl::haveOpenCL()) + ocl_detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); + else detect_all(imageForDetector, image_blurred, tmpCandidates, detectorResults, detect_flgs, trackers); - //else - //DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); - //printf("BOOOLZZZ %d\n", detect_flgs[0]); - //printf("BOOOLXXX %d\n", detect_flgs[1]); for (int k = 0; k < targetNum; k++) { //TLD Tracker data extraction @@ -95,7 +97,6 @@ namespace cv tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); Ptr data = tracker->data; - /////// data->frameNum++; for (int i = 0; i < 2; i++) @@ -125,14 +126,9 @@ namespace cv trackerNeedsReInit[k] = false; } } - //printf("CanditateRes Size: %d \n", candidatesRes[k].size()); + std::vector::iterator it = std::max_element(candidatesRes[k].begin(), candidatesRes[k].end()); - //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); - //for( int i = 0; i < (int)candidatesRes.size(); i++ ) - //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); - //data->printme(); - //tldModel->printme(stdout); if (it == candidatesRes[k].end()) { @@ -445,9 +441,260 @@ namespace cv //printf("%d %f %f\n", k, srValue, scValue); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); + + + if (maxSc < 0) + detect_flgs[k] = false; + else + { + res[k] = maxScRect; + //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); + detect_flgs[k] = true; + } + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); + } + + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, + std::vector> &trackers) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[0]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Size initSize = tldModel->getMinSize(); + + for (int k = 0; k < trackers.size(); k++) + patches[k].clear(); + + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + Mat tmp; + int dx = initSize.width / 10, dy = initSize.height / 10; + Size2d size = img.size(); + double scale = 1.0; + int npos = 0, nneg = 0; + double maxSc = -5.0; + Rect2d maxScRect; + int scaleID; + std::vector resized_imgs, blurred_imgs; + + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + + std::vector tmpP; + std::vector tmpI; + + //int64 e1, e2; + //double t; + //e1 = getTickCount(); + + //Detection part + //Generate windows and filter by variance + scaleID = 0; + resized_imgs.push_back(img); + blurred_imgs.push_back(imgBlurred); + do + { + Mat_ intImgP, intImgP2; + tld::TLDDetector::computeIntegralImages(resized_imgs[scaleID], intImgP, intImgP2); + for (int i = 0, imax = cvFloor((0.0 + resized_imgs[scaleID].cols - initSize.width) / dx); i < imax; i++) + { + for (int j = 0, jmax = cvFloor((0.0 + resized_imgs[scaleID].rows - initSize.height) / dy); j < jmax; j++) + { + //Optimized variance calculation + int x = dx * i, + y = dy * j, + width = initSize.width, + height = initSize.height; + double p = 0, p2 = 0; + double A, B, C, D; + + A = intImgP(y, x); + B = intImgP(y, x + width); + C = intImgP(y + height, x); + D = intImgP(y + height, x + width); + p = (A + D - B - C) / (width * height); + + A = intImgP2(y, x); + B = intImgP2(y, x + width); + C = intImgP2(y + height, x); + D = intImgP2(y + height, x + width); + p2 = (A + D - B - C) / (width * height); + double windowVar = p2 - p * p; + + //Loop for on all objects + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + //Optimized variance calculation + bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); + + if (!varPass) + continue; + varBuffer[k].push_back(Point(dx * i, dy * j)); + varScaleIDs[k].push_back(scaleID); + + //Debug display candidates after Variance Filter + double curScale = pow(tld::SCALE_STEP, scaleID); + debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); + } + } + } + scaleID++; + size.width /= tld::SCALE_STEP; + size.height /= tld::SCALE_STEP; + scale *= tld::SCALE_STEP; + resize(img, tmp, size, 0, 0, tld::DOWNSCALE_MODE); + resized_imgs.push_back(tmp); + GaussianBlur(resized_imgs[scaleID], tmp, tld::GaussBlurKernelSize, 0.0f); + blurred_imgs.push_back(tmp); + } while (size.width >= initSize.width && size.height >= initSize.height); + + + + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("Variance: %d\t%f\n", varBuffer.size(), t); + + //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); + + //Encsemble classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + + + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + + double ensRes = 0; + uchar* data = &blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x); + for (int x = 0; x < (int)tldModel->detector->classifiers.size(); x++) + { + int position = 0; + for (int n = 0; n < (int)tldModel->detector->classifiers[x].measurements.size(); n++) + { + position = position << 1; + if (data[tldModel->detector->classifiers[x].offset[n].x] < data[tldModel->detector->classifiers[x].offset[n].y]) + position++; + } + double posNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].x; + double negNum = (double)tldModel->detector->classifiers[x].posAndNeg[position].y; + if (posNum == 0.0 && negNum == 0.0) + continue; + else + ensRes += posNum / (posNum + negNum); + } + ensRes /= tldModel->detector->classifiers.size(); + ensRes = tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)); + + if (ensRes <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + /* + for (int i = 0; i < (int)varBuffer[k].size(); i++) + { + tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); + if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) + continue; + ensBuffer[k].push_back(varBuffer[k][i]); + ensScaleIDs[k].push_back(varScaleIDs[k][i]); + } + */ + } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + + //printf("varBuffer 1: %d\n", varBuffer[0].size()); + //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); + + //printf("varBuffer 2: %d\n", varBuffer[1].size()); + //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); + + //NN classification + //e1 = getTickCount(); + for (int k = 0; k < trackers.size(); k++) + { + //TLD Tracker data extraction + Tracker* trackerPtr = trackers[k]; + cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + //TLD Model Extraction + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + //Size InitSize = tldModel->getMinSize(); + npos = 0; + nneg = 0; + maxSc = -5.0; + + //Prepare batch of patches + int numOfPatches = (int)ensBuffer[k].size(); + Mat_ stdPatches(numOfPatches, 225); + double *resultSr = new double[numOfPatches]; + double *resultSc = new double[numOfPatches]; + + uchar *patchesData = stdPatches.data; + for (int i = 0; i < (int)ensBuffer.size(); i++) + { + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + uchar *stdPatchData = standardPatch.data; + for (int j = 0; j < 225; j++) + patchesData[225 * i + j] = stdPatchData[j]; + } + //Calculate Sr and Sc batches + tldModel->detector->ocl_batchSrSc(stdPatches, resultSr, resultSc, numOfPatches); + + for (int i = 0; i < (int)ensBuffer[k].size(); i++) + { + tld::TLDDetector::LabeledPatch labPatch; + standardPatch.data = &stdPatches.data[225 * i]; + double curScale = pow(tld::SCALE_STEP, ensScaleIDs[k][i]); + labPatch.rect = Rect2d(ensBuffer[k][i].x*curScale, ensBuffer[k][i].y*curScale, initSize.width * curScale, initSize.height * curScale); + tld::resample(resized_imgs[ensScaleIDs[k][i]], Rect2d(ensBuffer[k][i], initSize), standardPatch); + + double srValue, scValue; + srValue = resultSr[i]; + + ////To fix: Check the paper, probably this cause wrong learning + // + labPatch.isObject = srValue > tld::THETA_NN; + labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1; + patches[k].push_back(labPatch); + // + + if (!labPatch.isObject) + { + nneg++; + continue; + } + else + { + npos++; + } + scValue = resultSc[i]; + if (scValue > maxSc) + { + maxSc = scValue; + maxScRect = labPatch.rect; + } + //printf("%d %f %f\n", k, srValue, scValue); + } + + if (maxSc < 0) detect_flgs[k] = false; @@ -458,5 +705,9 @@ namespace cv detect_flgs[k] = true; } } + //e2 = getTickCount(); + //t = (e2 - e1) / getTickFrequency()*1000.0; + //printf("NN: %d\t%f\n", patches.size(), t); } + } \ No newline at end of file diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 2af2d8e8e..760cf9887 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -129,7 +129,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) if (i == 1) { - if (!ocl::haveOpenCL()) + if (ocl::haveOpenCL()) DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); else DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); From 365722e5e9c2ab3295df696b8ce810014154ce5d Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 19:50:04 +0900 Subject: [PATCH 24/32] Fixed spaces in >> --- modules/tracking/src/multiTracker.cpp | 22 +++++++++++----------- modules/tracking/src/multiTracker.hpp | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index d5fef17c9..0e6133ce0 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -71,9 +71,9 @@ namespace cv //best overlap around 92% Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); - std::vector> detectorResults(targetNum); - std::vector> candidates(targetNum); - std::vector> candidatesRes(targetNum); + std::vector > detectorResults(targetNum); + std::vector > candidates(targetNum); + std::vector > candidatesRes(targetNum); std::vector tmpCandidates(targetNum); std::vector detect_flgs(targetNum); std::vector trackerNeedsReInit(targetNum); @@ -221,8 +221,8 @@ namespace cv return true; } - void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, - std::vector> &trackers) + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, + std::vector > &trackers) { //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; @@ -245,8 +245,8 @@ namespace cv int scaleID; std::vector resized_imgs, blurred_imgs; - std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); - std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); std::vector tmpP; std::vector tmpI; @@ -457,8 +457,8 @@ namespace cv //printf("NN: %d\t%f\n", patches.size(), t); } - void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, std::vector &detect_flgs, - std::vector> &trackers) + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, + std::vector > &trackers) { //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; @@ -481,8 +481,8 @@ namespace cv int scaleID; std::vector resized_imgs, blurred_imgs; - std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); - std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); + std::vector > varBuffer(trackers.size()), ensBuffer(trackers.size()); + std::vector > varScaleIDs(trackers.size()), ensScaleIDs(trackers.size()); std::vector tmpP; std::vector tmpI; diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index a2b0a01ea..9dc65c0e3 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -7,10 +7,10 @@ namespace cv { - void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, - std::vector& detect_flgs, std::vector>& trackers); - void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch >> &patches, - std::vector& detect_flgs, std::vector>& trackers); + void detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, + std::vector& detect_flgs, std::vector >& trackers); + void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, + std::vector& detect_flgs, std::vector >& trackers); std::vector debugStack[10]; } #endif \ No newline at end of file From 6b58922d5f8f0cd31f50506f2a58acc7c50b813f Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 21:28:52 +0900 Subject: [PATCH 25/32] Removed inline specifiers --- modules/tracking/src/tldDetector.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 4dc433607..6e8878b01 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -73,8 +73,8 @@ namespace cv public: TLDDetector(){} ~TLDDetector(){} - inline double ensembleClassifierNum(const uchar* data); - inline void prepareClassifiers(int rowstep); + double ensembleClassifierNum(const uchar* data); + void prepareClassifiers(int rowstep); double Sr(const Mat_& patch); double ocl_Sr(const Mat_& patch); double Sc(const Mat_& patch); From 54f7c10a6919c65e852e9921842f9a728efe6c50 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 10 Aug 2015 22:04:06 +0900 Subject: [PATCH 26/32] Added OpenCV lincense and "precomp.hpp" --- modules/tracking/src/multiTracker.cpp | 41 ++++++++++++++++++++++++++ modules/tracking/src/multiTracker.hpp | 42 +++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 0e6133ce0..c3e314ea6 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,3 +1,44 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #include "multiTracker.hpp" namespace cv diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index 9dc65c0e3..c98d7df66 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -1,6 +1,48 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + #ifndef OPENCV_MULTITRACKER #define OPENCV_MULTITRACKER +#include "precomp.hpp" #include "tldTracker.hpp" #include "tldUtils.hpp" #include From 7e934bf182c1e574c9011ebc1fff5da1d22f1d79 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 16 Aug 2015 04:07:38 +0900 Subject: [PATCH 27/32] Warnings Fix #1 --- .../include/opencv2/tracking/tracker.hpp | 7 +- .../tracking/samples/multiTracker_test.cpp | 2 +- modules/tracking/src/multiTracker.cpp | 64 +++++++++---------- modules/tracking/src/tldDataset.cpp | 17 +++-- modules/tracking/src/tldTracker.cpp | 2 +- 5 files changed, 50 insertions(+), 42 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 8144e72bd..e94d0a30b 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1251,11 +1251,16 @@ class CV_EXPORTS_W MultiTracker { public: + MultiTracker() + { + targetNum = 0; + } + bool addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name); bool update(const Mat& image); - int targetNum = 0; + int targetNum; std::vector > trackers; std::vector boundingBoxes; std::vector colors; diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index c202a161f..204d54cc8 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -98,7 +98,7 @@ int main() // // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" // - char* tracker_algorithm_name = "TLD"; + const char* tracker_algorithm_name = "TLD"; Mat frame; paused = false; diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index c3e314ea6..cc80d67f9 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -76,7 +76,7 @@ namespace cv bool MultiTracker::update(const Mat& image) { printf("Naive-Loop MO-TLD Update....\n"); - for (int i = 0; i < trackers.size(); i++) + for (int i = 0; i < (int)trackers.size(); i++) if (!trackers[i]->update(image, boundingBoxes[i])) return false; @@ -132,11 +132,11 @@ namespace cv for (int k = 0; k < targetNum; k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); - Ptr data = tracker->data; + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + data = tracker->data; data->frameNum++; @@ -254,7 +254,7 @@ namespace cv //Debug display candidates after Variance Filter //////////////////////////////////////////////// Mat tmpImg = image; - for (int i = 0; i < debugStack[0].size(); i++) + for (int i = 0; i < (int)debugStack[0].size(); i++) //rectangle(tmpImg, debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); debugStack[0].clear(); tmpImg.copyTo(image); @@ -272,7 +272,7 @@ namespace cv tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); Size initSize = tldModel->getMinSize(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) patches[k].clear(); Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); @@ -331,13 +331,13 @@ namespace cv double windowVar = p2 - p * p; //Loop for on all objects - for (int k=0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); //Optimized variance calculation bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); @@ -373,13 +373,13 @@ namespace cv //Encsemble classification //e1 = getTickCount(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); for (int i = 0; i < (int)varBuffer[k].size(); i++) @@ -435,13 +435,13 @@ namespace cv //NN classification //e1 = getTickCount(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); npos = 0; nneg = 0; @@ -508,7 +508,7 @@ namespace cv tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); Size initSize = tldModel->getMinSize(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) patches[k].clear(); Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); @@ -567,13 +567,13 @@ namespace cv double windowVar = p2 - p * p; //Loop for on all objects - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); //Optimized variance calculation bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); @@ -609,13 +609,13 @@ namespace cv //Encsemble classification //e1 = getTickCount(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); for (int i = 0; i < (int)varBuffer[k].size(); i++) @@ -670,13 +670,13 @@ namespace cv //NN classification //e1 = getTickCount(); - for (int k = 0; k < trackers.size(); k++) + for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction - Tracker* trackerPtr = trackers[k]; - cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); + trackerPtr = trackers[k]; + tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); //Size InitSize = tldModel->getMinSize(); npos = 0; nneg = 0; diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 16d5596f2..240066a9c 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -51,7 +51,7 @@ namespace cv bool flagVOT = false; //TLD Dataset Parameters - char* tldFolderName[10] = { + const char* tldFolderName[10] = { "01_david", "02_jumping", "03_pedestrian1", @@ -63,7 +63,7 @@ namespace cv "09_carchase", "10_panda" }; - char* votFolderName[60] = { + const char* votFolderName[60] = { "bag", "ball1", "ball2", "basketball", "birds1", "birds2", "blanket", "bmx", "bolt1", "bolt2", "book", "butterfly", "car1", "car2", "crossing", "dinosaur", "fernando", "fish1", "fish2", "fish3", "fish4", "girl", "glove", "godfather", "graduate", "gymnastics1", "gymnastics2 ", "gymnastics3", "gymnastics4", "hand", @@ -72,11 +72,11 @@ namespace cv "singer2", "singer3", "soccer1", "soccer2", "soldier", "sphere", "tiger", "traffic", "tunnel", "wiper" }; - Rect2d tldInitBB[10] = { + const Rect2d tldInitBB[10] = { Rect2d(165, 93, 51, 54), Rect2d(147, 110, 33, 32), Rect2d(47, 51, 21, 36), Rect2d(130, 134, 21, 53), Rect2d(154, 102, 24, 52), Rect2d(142, 125, 90, 39), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(337, 219, 54, 37), Rect2d(58, 100, 27, 22) }; - Rect2d votInitBB[60] = { + const Rect2d votInitBB[60] = { Rect2d(142, 125, 90, 39), Rect2d(490, 400, 40, 40), Rect2d(273, 77, 27, 25), Rect2d(145, 84, 54, 37), Rect2d(58, 100, 27, 22), Rect2d(450, 380, 60, 60), Rect2d(290, 43, 23, 40), Rect2d(273, 77, 27, 25), Rect2d(225, 175, 50, 50), Rect2d(58, 100, 27, 22), @@ -118,7 +118,10 @@ namespace cv cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath, int datasetInd) { char* folderName = (char *)""; - int x, y, w, h; + int x = 0, + y = 0, + w = 0, + h = 0; //Index range // 1-10 TLD Dataset @@ -127,7 +130,7 @@ namespace cv if (datasetInd == 0) { - folderName = tldFolderName[id]; + folderName = (char*)tldFolderName[id]; x = tldInitBB[id].x; y = tldInitBB[id].y; w = tldInitBB[id].width; @@ -138,7 +141,7 @@ namespace cv } if (datasetInd == 1) { - folderName = votFolderName[id]; + folderName = (char*)votFolderName[id]; x = votInitBB[id].x; y = votInitBB[id].y; w = votInitBB[id].width; diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 760cf9887..5483fe336 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -238,7 +238,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) //Debug display candidates after Variance Filter //////////////////////////////////////////////// Mat tmpImg = image; - for (int i = 0; i < tldModel->detector->debugStack[0].size(); i++) + for (int i = 0; i < (int)tldModel->detector->debugStack[0].size(); i++) //rectangle(tmpImg, tldModel->detector->debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); tldModel->detector->debugStack[0].clear(); tmpImg.copyTo(image); From 167712b85d203ae41b12459031ad5860360080c5 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 16 Aug 2015 04:09:06 +0900 Subject: [PATCH 28/32] Warnings Fix #2 --- modules/tracking/samples/multiTracker_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/samples/multiTracker_test.cpp b/modules/tracking/samples/multiTracker_test.cpp index 204d54cc8..68cefc7e3 100644 --- a/modules/tracking/samples/multiTracker_test.cpp +++ b/modules/tracking/samples/multiTracker_test.cpp @@ -98,7 +98,7 @@ int main() // // "MIL", "BOOSTING", "MEDIANFLOW", "TLD" // - const char* tracker_algorithm_name = "TLD"; + char* tracker_algorithm_name = (char*)"TLD"; Mat frame; paused = false; From 190d2dcf85cbe08e4c128d5b734f8ae4ede48514 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Sun, 16 Aug 2015 04:25:09 +0900 Subject: [PATCH 29/32] Warnings Fix #3 --- modules/tracking/src/multiTracker.cpp | 2 -- modules/tracking/src/tldDataset.cpp | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index cc80d67f9..d7bb283e6 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -90,8 +90,6 @@ namespace cv printf("Optimized MO-TLD Update....\n"); //Get parameters from first object - //Set current target(tracker) parameters - Rect2d boundingBox = boundingBoxes[0]; //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); diff --git a/modules/tracking/src/tldDataset.cpp b/modules/tracking/src/tldDataset.cpp index 240066a9c..9ddbeed25 100644 --- a/modules/tracking/src/tldDataset.cpp +++ b/modules/tracking/src/tldDataset.cpp @@ -118,7 +118,7 @@ namespace cv cv::Rect2d tld_InitDataset(int videoInd, const char* rootPath, int datasetInd) { char* folderName = (char *)""; - int x = 0, + double x = 0, y = 0, w = 0, h = 0; From 4537103c3a20ee1a40620a1a089d77ba04230b2e Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 17 Aug 2015 03:08:38 +0900 Subject: [PATCH 30/32] Added Doxygen documentation --- .../include/opencv2/tracking/tracker.hpp | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index e94d0a30b..76b032a78 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -1247,28 +1247,80 @@ class CV_EXPORTS_W TrackerKCF : public Tracker /************************************ Multi-Tracker Classes ************************************/ +/** @brief Base abstract class for the long-term Multi Object Trackers: + +@sa Tracker, MultiTrackerTLD +*/ class CV_EXPORTS_W MultiTracker { public: - + /** @brief Constructor for Multitracker + */ MultiTracker() { targetNum = 0; } + /** @brief Add a new target to a tracking-list and initialize the tracker with a know bounding box that surrounding the target + @param image The initial frame + @param boundingBox The initial boundig box of target + @param tracker_algorithm_name Multi-tracker algorithm name + + @return True if new target initialization went succesfully, false otherwise + */ bool addTarget(const Mat& image, const Rect2d& boundingBox, char* tracker_algorithm_name); + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets + @param image The current frame + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ bool update(const Mat& image); + /** @brief Current number of targets in tracking-list + */ int targetNum; + + /** @brief Trackers list for Multi-Object-Tracker + */ std::vector > trackers; + + /** @brief Bounding Boxes list for Multi-Object-Tracker + */ std::vector boundingBoxes; + /** @brief List of randomly generated colors for bounding boxes display + */ std::vector colors; }; +/** @brief Multi Object Tracker for TLD. TLD is a novel tracking framework that explicitly decomposes +the long-term tracking task into tracking, learning and detection. + +The tracker follows the object from frame to frame. The detector localizes all appearances that +have been observed so far and corrects the tracker if necessary. The learning estimates detector’s +errors and updates it to avoid these errors in the future. The implementation is based on @cite TLD . + +The Median Flow algorithm (see cv::TrackerMedianFlow) was chosen as a tracking component in this +implementation, following authors. Tracker is supposed to be able to handle rapid motions, partial +occlusions, object absence etc. + +@sa Tracker, MultiTracker, TrackerTLD +*/ class CV_EXPORTS_W MultiTrackerTLD : public MultiTracker { public: + /** @brief Update all trackers from the tracking-list, find a new most likely bounding boxes for the targets by + optimized update method using some techniques to speedup calculations specifically for MO TLD. The only limitation + is that all target bounding boxes should have approximately same aspect ratios. Speed boost is around 20% + + @param image The current frame. + + @return True means that all targets were located and false means that tracker couldn't locate one of the targets in + current frame. Note, that latter *does not* imply that tracker has failed, maybe target is indeed + missing from the frame (say, out of sight) + */ bool update_opt(const Mat& image); }; From f16586f81b7ae278404d8feba2b5ce721cfa26d2 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 18 Aug 2015 06:47:03 +0900 Subject: [PATCH 31/32] Cleaning up debug prints/variables/flags/timers --- modules/tracking/src/multiTracker.cpp | 106 +------------ modules/tracking/src/multiTracker.hpp | 1 - modules/tracking/src/tldDetector.cpp | 209 +------------------------- modules/tracking/src/tldDetector.hpp | 1 - modules/tracking/src/tldModel.cpp | 44 ------ modules/tracking/src/tldTracker.cpp | 27 ---- 6 files changed, 6 insertions(+), 382 deletions(-) diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index d7bb283e6..78f0ee125 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -75,7 +75,6 @@ namespace cv bool MultiTracker::update(const Mat& image) { - printf("Naive-Loop MO-TLD Update....\n"); for (int i = 0; i < (int)trackers.size(); i++) if (!trackers[i]->update(image, boundingBoxes[i])) return false; @@ -87,8 +86,6 @@ namespace cv /*Optimized update method for TLD Multitracker */ bool MultiTrackerTLD::update_opt(const Mat& image) { - printf("Optimized MO-TLD Update....\n"); - //Get parameters from first object //TLD Tracker data extraction Tracker* trackerPtr = trackers[0]; @@ -186,16 +183,7 @@ namespace cv #if 1 if (it != candidatesRes[k].end()) - { tld::resample(imageForDetector, candidates[k][it - candidatesRes[k].begin()], standardPatch); - //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); - //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) - //dfprintf((stderr, "detector WON\n")); - } - else - { - //dfprintf((stderr, "%d x x\n", data->frameNum)); - } #endif if (*it > tld::CORE_THRESHOLD) @@ -226,7 +214,6 @@ namespace cv detectorResults[k][i].isObject = expertResult; } tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults[k]); - //dprintf(("%d relabeled by nExpert\n", negRelabeled)); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); if (ocl::haveOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); @@ -249,14 +236,7 @@ namespace cv } - //Debug display candidates after Variance Filter - //////////////////////////////////////////////// - Mat tmpImg = image; - for (int i = 0; i < (int)debugStack[0].size(); i++) - //rectangle(tmpImg, debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); - debugStack[0].clear(); - tmpImg.copyTo(image); - //////////////////////////////////////////////// + return true; } @@ -290,10 +270,6 @@ namespace cv std::vector tmpP; std::vector tmpI; - //int64 e1, e2; - //double t; - //e1 = getTickCount(); - //Detection part //Generate windows and filter by variance scaleID = 0; @@ -344,10 +320,6 @@ namespace cv continue; varBuffer[k].push_back(Point(dx * i, dy * j)); varScaleIDs[k].push_back(scaleID); - - //Debug display candidates after Variance Filter - double curScale = pow(tld::SCALE_STEP, scaleID); - debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); } } } @@ -361,16 +333,7 @@ namespace cv blurred_imgs.push_back(tmp); } while (size.width >= initSize.width && size.height >= initSize.height); - - - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Variance: %d\t%f\n", varBuffer.size(), t); - - //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); - //Encsemble classification - //e1 = getTickCount(); for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction @@ -410,29 +373,9 @@ namespace cv ensBuffer[k].push_back(varBuffer[k][i]); ensScaleIDs[k].push_back(varScaleIDs[k][i]); } - /* - for (int i = 0; i < (int)varBuffer[k].size(); i++) - { - tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); - if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) - continue; - ensBuffer[k].push_back(varBuffer[k][i]); - ensScaleIDs[k].push_back(varScaleIDs[k][i]); - } - */ } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); - - //printf("varBuffer 1: %d\n", varBuffer[0].size()); - //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); - - //printf("varBuffer 2: %d\n", varBuffer[1].size()); - //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); //NN classification - //e1 = getTickCount(); for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction @@ -477,7 +420,6 @@ namespace cv maxSc = scValue; maxScRect = labPatch.rect; } - //printf("%d %f %f\n", k, srValue, scValue); } @@ -487,13 +429,9 @@ namespace cv else { res[k] = maxScRect; - //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); detect_flgs[k] = true; } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); } void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector &detect_flgs, @@ -526,10 +464,6 @@ namespace cv std::vector tmpP; std::vector tmpI; - //int64 e1, e2; - //double t; - //e1 = getTickCount(); - //Detection part //Generate windows and filter by variance scaleID = 0; @@ -580,10 +514,6 @@ namespace cv continue; varBuffer[k].push_back(Point(dx * i, dy * j)); varScaleIDs[k].push_back(scaleID); - - //Debug display candidates after Variance Filter - double curScale = pow(tld::SCALE_STEP, scaleID); - debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, tldModel->getMinSize().width*curScale, tldModel->getMinSize().height*curScale)); } } } @@ -597,16 +527,7 @@ namespace cv blurred_imgs.push_back(tmp); } while (size.width >= initSize.width && size.height >= initSize.height); - - - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Variance: %d\t%f\n", varBuffer.size(), t); - - //printf("OrigVar 1: %f\n", *tldModel->detector->originalVariancePtr); - //Encsemble classification - //e1 = getTickCount(); for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction @@ -646,28 +567,9 @@ namespace cv ensBuffer[k].push_back(varBuffer[k][i]); ensScaleIDs[k].push_back(varScaleIDs[k][i]); } - /* - for (int i = 0; i < (int)varBuffer[k].size(); i++) - { - tldModel->detector->prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[k][i]].step[0])); - if (tldModel->detector->ensembleClassifierNum(&blurred_imgs[varScaleIDs[k][i]].at(varBuffer[k][i].y, varBuffer[k][i].x)) <= tld::ENSEMBLE_THRESHOLD) - continue; - ensBuffer[k].push_back(varBuffer[k][i]); - ensScaleIDs[k].push_back(varScaleIDs[k][i]); - } - */ } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - - //printf("varBuffer 1: %d\n", varBuffer[0].size()); - //printf("ensBuffer 1: %d\n", ensBuffer[0].size()); - - //printf("varBuffer 2: %d\n", varBuffer[1].size()); - //printf("ensBuffer 2: %d\n", ensBuffer[1].size()); //NN classification - //e1 = getTickCount(); for (int k = 0; k < (int)trackers.size(); k++) { //TLD Tracker data extraction @@ -675,7 +577,6 @@ namespace cv tracker = static_cast(trackerPtr); //TLD Model Extraction tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); - //Size InitSize = tldModel->getMinSize(); npos = 0; nneg = 0; maxSc = -5.0; @@ -730,7 +631,6 @@ namespace cv maxSc = scValue; maxScRect = labPatch.rect; } - //printf("%d %f %f\n", k, srValue, scValue); } @@ -740,13 +640,9 @@ namespace cv else { res[k] = maxScRect; - //printf("%f %f %f %f\n", maxScRect.x, maxScRect.y, maxScRect.width, maxScRect.height); detect_flgs[k] = true; } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); } } \ No newline at end of file diff --git a/modules/tracking/src/multiTracker.hpp b/modules/tracking/src/multiTracker.hpp index c98d7df66..9d363802c 100644 --- a/modules/tracking/src/multiTracker.hpp +++ b/modules/tracking/src/multiTracker.hpp @@ -53,6 +53,5 @@ namespace cv std::vector& detect_flgs, std::vector >& trackers); void ocl_detect_all(const Mat& img, const Mat& imgBlurred, std::vector& res, std::vector < std::vector < tld::TLDDetector::LabeledPatch > > &patches, std::vector& detect_flgs, std::vector >& trackers); - std::vector debugStack[10]; } #endif \ No newline at end of file diff --git a/modules/tracking/src/tldDetector.cpp b/modules/tracking/src/tldDetector.cpp index bfef69a45..ba9f03d9f 100644 --- a/modules/tracking/src/tldDetector.cpp +++ b/modules/tracking/src/tldDetector.cpp @@ -65,25 +65,6 @@ namespace cv // Calculate Relative similarity of the patch (NN-Model) double TLDDetector::Sr(const Mat_& patch) { - /* - int64 e1, e2; - float t; - e1 = getTickCount(); - double splus = 0.0, sminus = 0.0; - for (int i = 0; i < (int)(*positiveExamples).size(); i++) - splus = std::max(splus, 0.5 * (NCC((*positiveExamples)[i], patch) + 1.0)); - for (int i = 0; i < (int)(*negativeExamples).size(); i++) - sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); - e2 = getTickCount(); - t = (e2 - e1) / getTickFrequency()*1000.0; - printf("Sr: %f\n", t); - if (splus + sminus == 0.0) - return 0.0; - return splus / (sminus + splus); - */ - //int64 e1, e2; - //float t; - //e1 = getTickCount(); double splus = 0.0, sminus = 0.0; Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); for (int i = 0; i < *posNum; i++) @@ -96,9 +77,7 @@ namespace cv modelSample.data = &(negExp->data[i * 225]); sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Sr CPU: %f\n", t); + if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); @@ -106,10 +85,6 @@ namespace cv double TLDDetector::ocl_Sr(const Mat_& patch) { - //int64 e1, e2, e3, e4; - //double t; - //e1 = getTickCount(); - //e3 = getTickCount(); double splus = 0.0, sminus = 0.0; @@ -134,38 +109,12 @@ namespace cv *posNum, *negNum); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Mem Cpy GPU: %f\n", t); - size_t globSize = 1000; - //e3 = getTickCount(); + if (!k.run(1, &globSize, NULL, false)) printf("Kernel Run Error!!!"); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Kernel Run GPU: %f\n", t); - //e3 = getTickCount(); Mat resNCC = devNCC.getMat(ACCESS_READ); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Read Mem GPU: %f\n", t); - - ////Compare - //Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - //for (int i = 0; i < 200; i+=17) - //{ - // modelSample.data = &(posExp->data[i * 225]); - // printf("%f\t%f\n\n", resNCC.at(i), NCC(modelSample, patch)); - //} - - //for (int i = 0; i < 200; i+=23) - //{ - // modelSample.data = &(negExp->data[i * 225]); - // printf("%f\t%f\n", resNCC.at(500+i), NCC(modelSample, patch)); - //} - for (int i = 0; i < *posNum; i++) splus = std::max(splus, 0.5 * (resNCC.at(i) + 1.0)); @@ -173,10 +122,6 @@ namespace cv for (int i = 0; i < *negNum; i++) sminus = std::max(sminus, 0.5 * (resNCC.at(i+500) +1.0)); - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Sr GPU: %f\n\n", t); - if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); @@ -184,11 +129,6 @@ namespace cv void TLDDetector::ocl_batchSrSc(const Mat_& patches, double *resultSr, double *resultSc, int numOfPatches) { - //int64 e1, e2, e3, e4; - //double t; - //e1 = getTickCount(); - //e3 = getTickCount(); - UMat devPatches = patches.getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); UMat devPositiveSamples = posExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); UMat devNegativeSamples = negExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); @@ -212,25 +152,13 @@ namespace cv *negNum, numOfPatches); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Mem Cpy GPU: %f\n", t); - - // 2 -> Pos&Neg size_t globSize = 2 * numOfPatches*MAX_EXAMPLES_IN_MODEL; - //e3 = getTickCount(); + if (!k.run(1, &globSize, NULL, true)) printf("Kernel Run Error!!!"); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Kernel Run GPU: %f\n", t); - //e3 = getTickCount(); Mat posNCC = devPosNCC.getMat(ACCESS_READ); Mat negNCC = devNegNCC.getMat(ACCESS_READ); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Read Mem GPU: %f\n", t); //Calculate Srs for (int id = 0; id < numOfPatches; id++) @@ -256,62 +184,11 @@ namespace cv else resultSc[id] = spc / (smc + spc); } - - ////Compare positive NCCs - /*Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - Mat_ patch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - for (int j = 0; j < numOfPatches; j++) - { - for (int i = 0; i < 1; i++) - { - modelSample.data = &(posExp->data[i * 225]); - patch.data = &(patches.data[j * 225]); - printf("%f\t%f\n", resultSr[j], Sr(patch)); - printf("%f\t%f\n", resultSc[j], Sc(patch)); - } - }*/ - - //for (int i = 0; i < 200; i+=23) - //{ - // modelSample.data = &(negExp->data[i * 225]); - // printf("%f\t%f\n", resNCC.at(500+i), NCC(modelSample, patch)); - //} - - - - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Sr GPU: %f\n\n", t); } // Calculate Conservative similarity of the patch (NN-Model) double TLDDetector::Sc(const Mat_& patch) { - /* - int64 e1, e2; - float t; - e1 = getTickCount(); - double splus = 0.0, sminus = 0.0; - int med = getMedian((*timeStampsPositive)); - for (int i = 0; i < (int)(*positiveExamples).size(); i++) - { - if ((int)(*timeStampsPositive)[i] <= med) - splus = std::max(splus, 0.5 * (NCC((*positiveExamples)[i], patch) + 1.0)); - } - for (int i = 0; i < (int)(*negativeExamples).size(); i++) - sminus = std::max(sminus, 0.5 * (NCC((*negativeExamples)[i], patch) + 1.0)); - e2 = getTickCount(); - t = (e2 - e1) / getTickFrequency()*1000.0; - printf("Sc: %f\n", t); - if (splus + sminus == 0.0) - return 0.0; - - return splus / (sminus + splus); - */ - - //int64 e1, e2; - //double t; - //e1 = getTickCount(); double splus = 0.0, sminus = 0.0; Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); int med = getMedian((*timeStampsPositive)); @@ -328,9 +205,7 @@ namespace cv modelSample.data = &(negExp->data[i * 225]); sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0)); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Sc: %f\n", t); + if (splus + sminus == 0.0) return 0.0; @@ -339,13 +214,8 @@ namespace cv double TLDDetector::ocl_Sc(const Mat_& patch) { - //int64 e1, e2, e3, e4; - //float t; - //e1 = getTickCount(); double splus = 0.0, sminus = 0.0; - //e3 = getTickCount(); - UMat devPatch = patch.getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); UMat devPositiveSamples = posExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); UMat devNegativeSamples = negExp->getUMat(ACCESS_READ, USAGE_ALLOCATE_DEVICE_MEMORY); @@ -367,37 +237,12 @@ namespace cv *posNum, *negNum); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Mem Cpy GPU: %f\n", t); - size_t globSize = 1000; - //e3 = getTickCount(); + if (!k.run(1, &globSize, NULL, false)) printf("Kernel Run Error!!!"); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Kernel Run GPU: %f\n", t); - //e3 = getTickCount(); Mat resNCC = devNCC.getMat(ACCESS_READ); - //e4 = getTickCount(); - //t = (e4 - e3) / getTickFrequency()*1000.0; - //printf("Read Mem GPU: %f\n", t); - - ////Compare - //Mat_ modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); - //for (int i = 0; i < 200; i+=17) - //{ - // modelSample.data = &(posExp->data[i * 225]); - // printf("%f\t%f\n\n", resNCC.at(i), NCC(modelSample, patch)); - //} - - //for (int i = 0; i < 200; i+=23) - //{ - // modelSample.data = &(negExp->data[i * 225]); - // printf("%f\t%f\n", resNCC.at(500+i), NCC(modelSample, patch)); - //} int med = getMedian((*timeStampsPositive)); for (int i = 0; i < *posNum; i++) @@ -407,10 +252,6 @@ namespace cv for (int i = 0; i < *negNum; i++) sminus = std::max(sminus, 0.5 * (resNCC.at(i + 500) + 1.0)); - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Sc GPU: %f\n\n", t); - if (splus + sminus == 0.0) return 0.0; return splus / (sminus + splus); @@ -449,7 +290,6 @@ namespace cv break; } } - //dprintf(("%d rects in res\n", (int)res.size())); } //Detection - returns most probable new target location (Max Sc) @@ -469,10 +309,6 @@ namespace cv std::vector resized_imgs, blurred_imgs; std::vector varBuffer, ensBuffer; std::vector varScaleIDs, ensScaleIDs; - //int64 e1, e2; - //double t; - - //e1 = getTickCount(); //Detection part //Generate windows and filter by variance @@ -491,10 +327,6 @@ namespace cv continue; varBuffer.push_back(Point(dx * i, dy * j)); varScaleIDs.push_back(scaleID); - - //Debug display candidates after Variance Filter - double curScale = pow(tld::SCALE_STEP, scaleID); - debugStack[0].push_back(Rect2d(dx * i* curScale, dy * j*curScale, initSize.width*curScale, initSize.height*curScale)); } } scaleID++; @@ -506,12 +338,8 @@ namespace cv GaussianBlur(resized_imgs[scaleID], tmp, GaussBlurKernelSize, 0.0f); blurred_imgs.push_back(tmp); } while (size.width >= initSize.width && size.height >= initSize.height); - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Variance: %d\t%f\n", varBuffer.size(), t); //Encsemble classification - //e1 = getTickCount(); for (int i = 0; i < (int)varBuffer.size(); i++) { prepareClassifiers(static_cast (blurred_imgs[varScaleIDs[i]].step[0])); @@ -520,15 +348,8 @@ namespace cv ensBuffer.push_back(varBuffer[i]); ensScaleIDs.push_back(varScaleIDs[i]); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); - - //printf("varBuffer: %d\n", varBuffer.size()); - //printf("ensBuffer: %d\n", ensBuffer.size()); //NN classification - //e1 = getTickCount(); for (int i = 0; i < (int)ensBuffer.size(); i++) { LabeledPatch labPatch; @@ -562,9 +383,6 @@ namespace cv maxScRect = labPatch.rect; } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); if (maxSc < 0) return false; @@ -590,10 +408,7 @@ namespace cv std::vector resized_imgs, blurred_imgs; std::vector varBuffer, ensBuffer; std::vector varScaleIDs, ensScaleIDs; - //int64 e1, e2; - //double t; - //e1 = getTickCount(); //Detection part //Generate windows and filter by variance scaleID = 0; @@ -622,12 +437,8 @@ namespace cv GaussianBlur(resized_imgs[scaleID], tmp, GaussBlurKernelSize, 0.0f); blurred_imgs.push_back(tmp); } while (size.width >= initSize.width && size.height >= initSize.height); - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Variance: %d\t%f\n", varBuffer.size(), t); //Encsemble classification - //e1 = getTickCount(); for (int i = 0; i < (int)varBuffer.size(); i++) { prepareClassifiers((int)blurred_imgs[varScaleIDs[i]].step[0]); @@ -636,12 +447,8 @@ namespace cv ensBuffer.push_back(varBuffer[i]); ensScaleIDs.push_back(varScaleIDs[i]); } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("Ensemble: %d\t%f\n", ensBuffer.size(), t); //NN classification - //e1 = getTickCount(); //Prepare batch of patches int numOfPatches = (int)ensBuffer.size(); Mat_ stdPatches(numOfPatches, 225); @@ -671,9 +478,6 @@ namespace cv srValue = resultSr[i]; - //srValue = Sr(standardPatch); - //printf("%f\t%f\t\n", srValue, resultSr[i]); - ////To fix: Check the paper, probably this cause wrong learning // labPatch.isObject = srValue > THETA_NN; @@ -697,9 +501,6 @@ namespace cv maxScRect = labPatch.rect; } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency()*1000.0; - //printf("NN: %d\t%f\n", patches.size(), t); if (maxSc < 0) return false; diff --git a/modules/tracking/src/tldDetector.hpp b/modules/tracking/src/tldDetector.hpp index 6e8878b01..7f5a3f9e5 100644 --- a/modules/tracking/src/tldDetector.hpp +++ b/modules/tracking/src/tldDetector.hpp @@ -81,7 +81,6 @@ namespace cv double ocl_Sc(const Mat_& patch); void ocl_batchSrSc(const Mat_& patches, double *resultSr, double *resultSc, int numOfPatches); - std::vector debugStack[10]; std::vector classifiers; Mat *posExp, *negExp; int *posNum, *negNum; diff --git a/modules/tracking/src/tldModel.cpp b/modules/tracking/src/tldModel.cpp index 085402265..9415c2a99 100644 --- a/modules/tracking/src/tldModel.cpp +++ b/modules/tracking/src/tldModel.cpp @@ -140,7 +140,6 @@ namespace cv detector->classifiers[k].integrate(blurredPatch, false); } } - //dprintf(("positive patches: %d\nnegative patches: %d\n", (int)positiveExamples.size(), (int)negativeExamples.size())); } @@ -180,16 +179,6 @@ namespace cv detector->classifiers[i].integrate(blurredPatch, patches[k].isObject); } } - /* - if( negativeIntoModel > 0 ) - dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); - if( positiveIntoModel > 0) - dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); - if( negativeIntoEnsemble > 0 ) - dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); - if( positiveIntoEnsemble > 0 ) - dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); - dfprintf((stdout, "\n"));*/ } @@ -198,9 +187,6 @@ namespace cv int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; if ((int)eForModel.size() == 0) return; - //int64 e1, e2; - //double t; - //e1 = getTickCount(); for (int k = 0; k < (int)eForModel.size(); k++) { double sr = detector->Sr(eForModel[k]); @@ -231,19 +217,6 @@ namespace cv detector->classifiers[i].integrate(eForEnsemble[k], isPositive); } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency() * 1000; - //printf("Integrate Additional: %fms\n", t); - /* - if( negativeIntoModel > 0 ) - dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); - if( positiveIntoModel > 0 ) - dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); - if( negativeIntoEnsemble > 0 ) - dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); - if( positiveIntoEnsemble > 0 ) - dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); - dfprintf((stdout, "\n"));*/ } void TrackerTLDModel::ocl_integrateAdditional(const std::vector >& eForModel, const std::vector >& eForEnsemble, bool isPositive) @@ -251,10 +224,6 @@ namespace cv int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; if ((int)eForModel.size() == 0) return; - //int64 e1, e2; - //double t; - //e1 = getTickCount(); - //Prepare batch of patches int numOfPatches = (int)eForModel.size(); Mat_ stdPatches(numOfPatches, 225); @@ -301,19 +270,6 @@ namespace cv detector->classifiers[i].integrate(eForEnsemble[k], isPositive); } } - //e2 = getTickCount(); - //t = (e2 - e1) / getTickFrequency() * 1000; - //printf("Integrate Additional OCL: %fms\n", t); - /* - if( negativeIntoModel > 0 ) - dfprintf((stdout, "negativeIntoModel = %d ", negativeIntoModel)); - if( positiveIntoModel > 0 ) - dfprintf((stdout, "positiveIntoModel = %d ", positiveIntoModel)); - if( negativeIntoEnsemble > 0 ) - dfprintf((stdout, "negativeIntoEnsemble = %d ", negativeIntoEnsemble)); - if( positiveIntoEnsemble > 0 ) - dfprintf((stdout, "positiveIntoEnsemble = %d ", positiveIntoEnsemble)); - dfprintf((stdout, "\n"));*/ } //Push the patch to the model diff --git a/modules/tracking/src/tldTracker.cpp b/modules/tracking/src/tldTracker.cpp index 5483fe336..a1334a91c 100644 --- a/modules/tracking/src/tldTracker.cpp +++ b/modules/tracking/src/tldTracker.cpp @@ -151,12 +151,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) } std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); - //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); - //for( int i = 0; i < (int)candidatesRes.size(); i++ ) - //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); - //data->printme(); - //tldModel->printme(stdout); - if( it == candidatesRes.end() ) { data->confident = false; @@ -173,16 +167,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) #if 1 if( it != candidatesRes.end() ) - { resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); - //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); - //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) - //dfprintf((stderr, "detector WON\n")); - } - else - { - //dfprintf((stderr, "%d x x\n", data->frameNum)); - } #endif if( *it > CORE_THRESHOLD ) @@ -213,7 +198,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) detectorResults[i].isObject = expertResult; } tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); - //dprintf(("%d relabeled by nExpert\n", negRelabeled)); pExpert.additionalExamples(examplesForModel, examplesForEnsemble); if (ocl::haveOpenCL()) tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); @@ -234,16 +218,6 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox) #endif } - - //Debug display candidates after Variance Filter - //////////////////////////////////////////////// - Mat tmpImg = image; - for (int i = 0; i < (int)tldModel->detector->debugStack[0].size(); i++) - //rectangle(tmpImg, tldModel->detector->debugStack[0][i], Scalar(255, 255, 255), 1, 1, 0); - tldModel->detector->debugStack[0].clear(); - tmpImg.copyTo(image); - - //////////////////////////////////////////////// return true; } @@ -310,7 +284,6 @@ Data::Data(Rect2d initBox) minSize.width = (int)(initBox.width * 20.0 / minDim); minSize.height = (int)(initBox.height * 20.0 / minDim); frameNum = 0; - //dprintf(("minSize = %dx%d\n", minSize.width, minSize.height)); } void Data::printme(FILE* port) From 94447d516099c3951d115fa429668b7e8563d230 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 18 Aug 2015 15:03:46 +0900 Subject: [PATCH 32/32] Restored *protected:*' in tracker.hpp --- .../include/opencv2/tracking/tracker.hpp | 7 ++++++- modules/tracking/src/multiTracker.cpp | 20 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index 76b032a78..04660d6a6 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -565,7 +565,12 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm virtual void read( const FileNode& fn )=0; virtual void write( FileStorage& fs ) const=0; - public: + Ptr getModel() + { + return model; + } + + protected: virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index 78f0ee125..3053f2174 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -91,7 +91,7 @@ namespace cv Tracker* trackerPtr = trackers[0]; tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Ptr data = tracker->data; double scale = data->getScale(); @@ -130,7 +130,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); data = tracker->data; data->frameNum++; @@ -247,7 +247,7 @@ namespace cv Tracker* trackerPtr = trackers[0]; cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); for (int k = 0; k < (int)trackers.size(); k++) @@ -311,7 +311,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); //Optimized variance calculation bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); @@ -340,7 +340,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); for (int i = 0; i < (int)varBuffer[k].size(); i++) @@ -382,7 +382,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); npos = 0; nneg = 0; @@ -441,7 +441,7 @@ namespace cv Tracker* trackerPtr = trackers[0]; cv::tld::TrackerTLDImpl* tracker = static_cast(trackerPtr); //TLD Model Extraction - tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); Size initSize = tldModel->getMinSize(); for (int k = 0; k < (int)trackers.size(); k++) @@ -505,7 +505,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); //Optimized variance calculation bool varPass = (windowVar > tld::VARIANCE_THRESHOLD * *tldModel->detector->originalVariancePtr); @@ -534,7 +534,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); for (int i = 0; i < (int)varBuffer[k].size(); i++) @@ -576,7 +576,7 @@ namespace cv trackerPtr = trackers[k]; tracker = static_cast(trackerPtr); //TLD Model Extraction - tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->getModel())); npos = 0; nneg = 0; maxSc = -5.0;