From 697793090d2359b56074df17fdf4645043eba30a Mon Sep 17 00:00:00 2001 From: Vladislav Vinogradov Date: Tue, 7 May 2013 11:02:59 +0400 Subject: [PATCH] refactored GMG algorithm --- .../gpubgsegm/include/opencv2/gpubgsegm.hpp | 85 +----- modules/gpubgsegm/perf/perf_bgsegm.cpp | 13 +- modules/gpubgsegm/src/cuda/gmg.cu | 2 +- modules/gpubgsegm/src/gmg.cpp | 258 ++++++++++++------ modules/gpubgsegm/src/precomp.hpp | 2 + modules/gpubgsegm/test/test_bgsegm.cpp | 13 +- samples/gpu/bgfg_segm.cpp | 47 ++-- 7 files changed, 234 insertions(+), 186 deletions(-) diff --git a/modules/gpubgsegm/include/opencv2/gpubgsegm.hpp b/modules/gpubgsegm/include/opencv2/gpubgsegm.hpp index 2b64f6e409..d686b3c9c8 100644 --- a/modules/gpubgsegm/include/opencv2/gpubgsegm.hpp +++ b/modules/gpubgsegm/include/opencv2/gpubgsegm.hpp @@ -91,6 +91,20 @@ CV_EXPORTS Ptr createBackgroundSubtractorMOG2(int history = 500, double varThreshold = 16, bool detectShadows = true); +//////////////////////////////////////////////////// +// GMG + +class CV_EXPORTS BackgroundSubtractorGMG : public cv::BackgroundSubtractorGMG +{ +public: + using cv::BackgroundSubtractorGMG::apply; + + virtual void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream) = 0; +}; + +CV_EXPORTS Ptr + createBackgroundSubtractorGMG(int initializationFrames = 120, double decisionThreshold = 0.8); + @@ -161,77 +175,6 @@ private: std::auto_ptr impl_; }; -/** - * Background Subtractor module. Takes a series of images and returns a sequence of mask (8UC1) - * images of the same size, where 255 indicates Foreground and 0 represents Background. - * This class implements an algorithm described in "Visual Tracking of Human Visitors under - * Variable-Lighting Conditions for a Responsive Audio Art Installation," A. Godbehere, - * A. Matsukawa, K. Goldberg, American Control Conference, Montreal, June 2012. - */ -class CV_EXPORTS GMG_GPU -{ -public: - GMG_GPU(); - - /** - * Validate parameters and set up data structures for appropriate frame size. - * @param frameSize Input frame size - * @param min Minimum value taken on by pixels in image sequence. Usually 0 - * @param max Maximum value taken on by pixels in image sequence. e.g. 1.0 or 255 - */ - void initialize(Size frameSize, float min = 0.0f, float max = 255.0f); - - /** - * Performs single-frame background subtraction and builds up a statistical background image - * model. - * @param frame Input frame - * @param fgmask Output mask image representing foreground and background pixels - * @param stream Stream for the asynchronous version - */ - void operator ()(const GpuMat& frame, GpuMat& fgmask, float learningRate = -1.0f, Stream& stream = Stream::Null()); - - //! Releases all inner buffers - void release(); - - //! Total number of distinct colors to maintain in histogram. - int maxFeatures; - - //! Set between 0.0 and 1.0, determines how quickly features are "forgotten" from histograms. - float learningRate; - - //! Number of frames of video to use to initialize histograms. - int numInitializationFrames; - - //! Number of discrete levels in each channel to be used in histograms. - int quantizationLevels; - - //! Prior probability that any given pixel is a background pixel. A sensitivity parameter. - float backgroundPrior; - - //! Value above which pixel is determined to be FG. - float decisionThreshold; - - //! Smoothing radius, in pixels, for cleaning up FG image. - int smoothingRadius; - - //! Perform background model update. - bool updateBackgroundModel; - -private: - float maxVal_, minVal_; - - Size frameSize_; - - int frameNum_; - - GpuMat nfeatures_; - GpuMat colors_; - GpuMat weights_; - - Ptr boxFilter_; - GpuMat buf_; -}; - }} // namespace cv { namespace gpu { #endif /* __OPENCV_GPUBGSEGM_HPP__ */ diff --git a/modules/gpubgsegm/perf/perf_bgsegm.cpp b/modules/gpubgsegm/perf/perf_bgsegm.cpp index 8ec9c6f026..9fd3811e37 100644 --- a/modules/gpubgsegm/perf/perf_bgsegm.cpp +++ b/modules/gpubgsegm/perf/perf_bgsegm.cpp @@ -462,10 +462,10 @@ PERF_TEST_P(Video_Cn_MaxFeatures, GMG, cv::gpu::GpuMat d_frame(frame); cv::gpu::GpuMat foreground; - cv::gpu::GMG_GPU d_gmg; - d_gmg.maxFeatures = maxFeatures; + cv::Ptr d_gmg = cv::gpu::createBackgroundSubtractorGMG(); + d_gmg->setMaxFeatures(maxFeatures); - d_gmg(d_frame, foreground); + d_gmg->apply(d_frame, foreground); for (int i = 0; i < 150; ++i) { @@ -490,7 +490,7 @@ PERF_TEST_P(Video_Cn_MaxFeatures, GMG, d_frame.upload(frame); startTimer(); next(); - d_gmg(d_frame, foreground); + d_gmg->apply(d_frame, foreground); stopTimer(); } @@ -501,9 +501,8 @@ PERF_TEST_P(Video_Cn_MaxFeatures, GMG, cv::Mat foreground; cv::Mat zeros(frame.size(), CV_8UC1, cv::Scalar::all(0)); - cv::Ptr gmg = cv::createBackgroundSubtractorGMG(); - gmg->set("maxFeatures", maxFeatures); - //gmg.initialize(frame.size(), 0.0, 255.0); + cv::Ptr gmg = cv::createBackgroundSubtractorGMG(); + gmg->setMaxFeatures(maxFeatures); gmg->apply(frame, foreground); diff --git a/modules/gpubgsegm/src/cuda/gmg.cu b/modules/gpubgsegm/src/cuda/gmg.cu index 8ae9b037b2..235c1f0e2e 100644 --- a/modules/gpubgsegm/src/cuda/gmg.cu +++ b/modules/gpubgsegm/src/cuda/gmg.cu @@ -47,7 +47,7 @@ #include "opencv2/core/cuda/limits.hpp" namespace cv { namespace gpu { namespace cudev { - namespace bgfg_gmg + namespace gmg { __constant__ int c_width; __constant__ int c_height; diff --git a/modules/gpubgsegm/src/gmg.cpp b/modules/gpubgsegm/src/gmg.cpp index b97f0836f4..e11d1bd41c 100644 --- a/modules/gpubgsegm/src/gmg.cpp +++ b/modules/gpubgsegm/src/gmg.cpp @@ -42,17 +42,17 @@ #include "precomp.hpp" -#if !defined HAVE_CUDA || defined(CUDA_DISABLER) +using namespace cv; +using namespace cv::gpu; -cv::gpu::GMG_GPU::GMG_GPU() { throw_no_cuda(); } -void cv::gpu::GMG_GPU::initialize(cv::Size, float, float) { throw_no_cuda(); } -void cv::gpu::GMG_GPU::operator ()(const cv::gpu::GpuMat&, cv::gpu::GpuMat&, float, cv::gpu::Stream&) { throw_no_cuda(); } -void cv::gpu::GMG_GPU::release() {} +#if !defined HAVE_CUDA || defined(CUDA_DISABLER) || !defined(HAVE_OPENCV_GPUFILTERS) + +Ptr cv::gpu::createBackgroundSubtractorGMG(int, double) { throw_no_cuda(); return Ptr(); } #else namespace cv { namespace gpu { namespace cudev { - namespace bgfg_gmg + namespace gmg { void loadConstants(int width, int height, float minVal, float maxVal, int quantizationLevels, float backgroundPrior, float decisionThreshold, int maxFeatures, int numInitializationFrames); @@ -63,103 +63,209 @@ namespace cv { namespace gpu { namespace cudev { } }}} -cv::gpu::GMG_GPU::GMG_GPU() +namespace { - maxFeatures = 64; - learningRate = 0.025f; - numInitializationFrames = 120; - quantizationLevels = 16; - backgroundPrior = 0.8f; - decisionThreshold = 0.8f; - smoothingRadius = 7; - updateBackgroundModel = true; -} + class GMGImpl : public gpu::BackgroundSubtractorGMG + { + public: + GMGImpl(int initializationFrames, double decisionThreshold); -void cv::gpu::GMG_GPU::initialize(cv::Size frameSize, float min, float max) -{ - using namespace cv::gpu::cudev::bgfg_gmg; + void apply(InputArray image, OutputArray fgmask, double learningRate=-1); + void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream); - CV_Assert(min < max); - CV_Assert(maxFeatures > 0); - CV_Assert(learningRate >= 0.0f && learningRate <= 1.0f); - CV_Assert(numInitializationFrames >= 1); - CV_Assert(quantizationLevels >= 1 && quantizationLevels <= 255); - CV_Assert(backgroundPrior >= 0.0f && backgroundPrior <= 1.0f); + void getBackgroundImage(OutputArray backgroundImage) const; - minVal_ = min; - maxVal_ = max; + int getMaxFeatures() const { return maxFeatures_; } + void setMaxFeatures(int maxFeatures) { maxFeatures_ = maxFeatures; } - frameSize_ = frameSize; + double getDefaultLearningRate() const { return learningRate_; } + void setDefaultLearningRate(double lr) { learningRate_ = (float) lr; } - frameNum_ = 0; + int getNumFrames() const { return numInitializationFrames_; } + void setNumFrames(int nframes) { numInitializationFrames_ = nframes; } - nfeatures_.create(frameSize_, CV_32SC1); - colors_.create(maxFeatures * frameSize_.height, frameSize_.width, CV_32SC1); - weights_.create(maxFeatures * frameSize_.height, frameSize_.width, CV_32FC1); + int getQuantizationLevels() const { return quantizationLevels_; } + void setQuantizationLevels(int nlevels) { quantizationLevels_ = nlevels; } - nfeatures_.setTo(cv::Scalar::all(0)); + double getBackgroundPrior() const { return backgroundPrior_; } + void setBackgroundPrior(double bgprior) { backgroundPrior_ = (float) bgprior; } - if (smoothingRadius > 0) - boxFilter_ = cv::gpu::createBoxFilter(CV_8UC1, -1, cv::Size(smoothingRadius, smoothingRadius)); + int getSmoothingRadius() const { return smoothingRadius_; } + void setSmoothingRadius(int radius) { smoothingRadius_ = radius; } - loadConstants(frameSize_.width, frameSize_.height, minVal_, maxVal_, quantizationLevels, backgroundPrior, decisionThreshold, maxFeatures, numInitializationFrames); -} + double getDecisionThreshold() const { return decisionThreshold_; } + void setDecisionThreshold(double thresh) { decisionThreshold_ = (float) thresh; } -void cv::gpu::GMG_GPU::operator ()(const cv::gpu::GpuMat& frame, cv::gpu::GpuMat& fgmask, float newLearningRate, cv::gpu::Stream& stream) -{ - using namespace cv::gpu::cudev::bgfg_gmg; + bool getUpdateBackgroundModel() const { return updateBackgroundModel_; } + void setUpdateBackgroundModel(bool update) { updateBackgroundModel_ = update; } - typedef void (*func_t)(PtrStepSzb frame, PtrStepb fgmask, PtrStepSzi colors, PtrStepf weights, PtrStepi nfeatures, - int frameNum, float learningRate, bool updateBackgroundModel, cudaStream_t stream); - static const func_t funcs[6][4] = - { - {update_gpu, 0, update_gpu, update_gpu}, - {0,0,0,0}, - {update_gpu, 0, update_gpu, update_gpu}, - {0,0,0,0}, - {0,0,0,0}, - {update_gpu, 0, update_gpu, update_gpu} + double getMinVal() const { return minVal_; } + void setMinVal(double val) { minVal_ = (float) val; } + + double getMaxVal() const { return maxVal_; } + void setMaxVal(double val) { maxVal_ = (float) val; } + + private: + void initialize(Size frameSize, float min, float max); + + //! Total number of distinct colors to maintain in histogram. + int maxFeatures_; + + //! Set between 0.0 and 1.0, determines how quickly features are "forgotten" from histograms. + float learningRate_; + + //! Number of frames of video to use to initialize histograms. + int numInitializationFrames_; + + //! Number of discrete levels in each channel to be used in histograms. + int quantizationLevels_; + + //! Prior probability that any given pixel is a background pixel. A sensitivity parameter. + float backgroundPrior_; + + //! Smoothing radius, in pixels, for cleaning up FG image. + int smoothingRadius_; + + //! Value above which pixel is determined to be FG. + float decisionThreshold_; + + //! Perform background model update. + bool updateBackgroundModel_; + + float minVal_, maxVal_; + + Size frameSize_; + int frameNum_; + + GpuMat nfeatures_; + GpuMat colors_; + GpuMat weights_; + + Ptr boxFilter_; + GpuMat buf_; }; - CV_Assert(frame.depth() == CV_8U || frame.depth() == CV_16U || frame.depth() == CV_32F); - CV_Assert(frame.channels() == 1 || frame.channels() == 3 || frame.channels() == 4); + GMGImpl::GMGImpl(int initializationFrames, double decisionThreshold) + { + maxFeatures_ = 64; + learningRate_ = 0.025f; + numInitializationFrames_ = initializationFrames; + quantizationLevels_ = 16; + backgroundPrior_ = 0.8f; + decisionThreshold_ = (float) decisionThreshold; + smoothingRadius_ = 7; + updateBackgroundModel_ = true; + minVal_ = maxVal_ = 0; + } - if (newLearningRate != -1.0f) + void GMGImpl::apply(InputArray image, OutputArray fgmask, double learningRate) { - CV_Assert(newLearningRate >= 0.0f && newLearningRate <= 1.0f); - learningRate = newLearningRate; + apply(image, fgmask, learningRate, Stream::Null()); } - if (frame.size() != frameSize_) - initialize(frame.size(), 0.0f, frame.depth() == CV_8U ? 255.0f : frame.depth() == CV_16U ? std::numeric_limits::max() : 1.0f); + void GMGImpl::apply(InputArray _frame, OutputArray _fgmask, double newLearningRate, Stream& stream) + { + using namespace cv::gpu::cudev::gmg; + + typedef void (*func_t)(PtrStepSzb frame, PtrStepb fgmask, PtrStepSzi colors, PtrStepf weights, PtrStepi nfeatures, + int frameNum, float learningRate, bool updateBackgroundModel, cudaStream_t stream); + static const func_t funcs[6][4] = + { + {update_gpu, 0, update_gpu, update_gpu}, + {0,0,0,0}, + {update_gpu, 0, update_gpu, update_gpu}, + {0,0,0,0}, + {0,0,0,0}, + {update_gpu, 0, update_gpu, update_gpu} + }; + + GpuMat frame = _frame.getGpuMat(); + + CV_Assert( frame.depth() == CV_8U || frame.depth() == CV_16U || frame.depth() == CV_32F ); + CV_Assert( frame.channels() == 1 || frame.channels() == 3 || frame.channels() == 4 ); + + if (newLearningRate != -1.0) + { + CV_Assert( newLearningRate >= 0.0 && newLearningRate <= 1.0 ); + learningRate_ = (float) newLearningRate; + } + + if (frame.size() != frameSize_) + { + double minVal = minVal_; + double maxVal = maxVal_; - fgmask.create(frameSize_, CV_8UC1); - fgmask.setTo(cv::Scalar::all(0), stream); + if (minVal_ == 0 && maxVal_ == 0) + { + minVal = 0; + maxVal = frame.depth() == CV_8U ? 255.0 : frame.depth() == CV_16U ? std::numeric_limits::max() : 1.0; + } - funcs[frame.depth()][frame.channels() - 1](frame, fgmask, colors_, weights_, nfeatures_, frameNum_, learningRate, updateBackgroundModel, cv::gpu::StreamAccessor::getStream(stream)); + initialize(frame.size(), (float) minVal, (float) maxVal); + } + + _fgmask.create(frameSize_, CV_8UC1); + GpuMat fgmask = _fgmask.getGpuMat(); + + fgmask.setTo(Scalar::all(0), stream); + + funcs[frame.depth()][frame.channels() - 1](frame, fgmask, colors_, weights_, nfeatures_, frameNum_, + learningRate_, updateBackgroundModel_, StreamAccessor::getStream(stream)); + + // medianBlur + if (smoothingRadius_ > 0) + { + boxFilter_->apply(fgmask, buf_, stream); + const int minCount = (smoothingRadius_ * smoothingRadius_ + 1) / 2; + const double thresh = 255.0 * minCount / (smoothingRadius_ * smoothingRadius_); + gpu::threshold(buf_, fgmask, thresh, 255.0, THRESH_BINARY, stream); + } + + // keep track of how many frames we have processed + ++frameNum_; + } - // medianBlur - if (smoothingRadius > 0) + void GMGImpl::getBackgroundImage(OutputArray backgroundImage) const { - boxFilter_->apply(fgmask, buf_, stream); - int minCount = (smoothingRadius * smoothingRadius + 1) / 2; - double thresh = 255.0 * minCount / (smoothingRadius * smoothingRadius); - cv::gpu::threshold(buf_, fgmask, thresh, 255.0, cv::THRESH_BINARY, stream); + (void) backgroundImage; + CV_Error(Error::StsNotImplemented, "Not implemented"); } - // keep track of how many frames we have processed - ++frameNum_; + void GMGImpl::initialize(Size frameSize, float min, float max) + { + using namespace cv::gpu::cudev::gmg; + + CV_Assert( maxFeatures_ > 0 ); + CV_Assert( learningRate_ >= 0.0f && learningRate_ <= 1.0f); + CV_Assert( numInitializationFrames_ >= 1); + CV_Assert( quantizationLevels_ >= 1 && quantizationLevels_ <= 255); + CV_Assert( backgroundPrior_ >= 0.0f && backgroundPrior_ <= 1.0f); + + minVal_ = min; + maxVal_ = max; + CV_Assert( minVal_ < maxVal_ ); + + frameSize_ = frameSize; + + frameNum_ = 0; + + nfeatures_.create(frameSize_, CV_32SC1); + colors_.create(maxFeatures_ * frameSize_.height, frameSize_.width, CV_32SC1); + weights_.create(maxFeatures_ * frameSize_.height, frameSize_.width, CV_32FC1); + + nfeatures_.setTo(Scalar::all(0)); + + if (smoothingRadius_ > 0) + boxFilter_ = gpu::createBoxFilter(CV_8UC1, -1, Size(smoothingRadius_, smoothingRadius_)); + + loadConstants(frameSize_.width, frameSize_.height, minVal_, maxVal_, + quantizationLevels_, backgroundPrior_, decisionThreshold_, maxFeatures_, numInitializationFrames_); + } } -void cv::gpu::GMG_GPU::release() +Ptr cv::gpu::createBackgroundSubtractorGMG(int initializationFrames, double decisionThreshold) { - frameSize_ = Size(); - - nfeatures_.release(); - colors_.release(); - weights_.release(); - boxFilter_.release(); - buf_.release(); + return new GMGImpl(initializationFrames, decisionThreshold); } #endif diff --git a/modules/gpubgsegm/src/precomp.hpp b/modules/gpubgsegm/src/precomp.hpp index 5f120961b2..f3aa637e48 100644 --- a/modules/gpubgsegm/src/precomp.hpp +++ b/modules/gpubgsegm/src/precomp.hpp @@ -52,4 +52,6 @@ #include "opencv2/core/private.gpu.hpp" +#include "opencv2/opencv_modules.hpp" + #endif /* __OPENCV_PRECOMP_H__ */ diff --git a/modules/gpubgsegm/test/test_bgsegm.cpp b/modules/gpubgsegm/test/test_bgsegm.cpp index 4ff505f9aa..41405c91f5 100644 --- a/modules/gpubgsegm/test/test_bgsegm.cpp +++ b/modules/gpubgsegm/test/test_bgsegm.cpp @@ -372,16 +372,15 @@ GPU_TEST_P(GMG, Accuracy) cv::Mat frame = randomMat(size, type, 0, 100); cv::gpu::GpuMat d_frame = loadMat(frame, useRoi); - cv::gpu::GMG_GPU gmg; - gmg.numInitializationFrames = 5; - gmg.smoothingRadius = 0; - gmg.initialize(d_frame.size(), 0, 255); + cv::Ptr gmg = cv::gpu::createBackgroundSubtractorGMG(); + gmg->setNumFrames(5); + gmg->setSmoothingRadius(0); cv::gpu::GpuMat d_fgmask = createMat(size, CV_8UC1, useRoi); - for (int i = 0; i < gmg.numInitializationFrames; ++i) + for (int i = 0; i < gmg->getNumFrames(); ++i) { - gmg(d_frame, d_fgmask); + gmg->apply(d_frame, d_fgmask); // fgmask should be entirely background during training ASSERT_MAT_NEAR(zeros, d_fgmask, 0); @@ -389,7 +388,7 @@ GPU_TEST_P(GMG, Accuracy) frame = randomMat(size, type, 160, 255); d_frame = loadMat(frame, useRoi); - gmg(d_frame, d_fgmask); + gmg->apply(d_frame, d_fgmask); // now fgmask should be entirely foreground ASSERT_MAT_NEAR(fullfg, d_fgmask, 0); diff --git a/samples/gpu/bgfg_segm.cpp b/samples/gpu/bgfg_segm.cpp index 01d8680f36..a9655b2ce4 100644 --- a/samples/gpu/bgfg_segm.cpp +++ b/samples/gpu/bgfg_segm.cpp @@ -18,10 +18,10 @@ using namespace cv::gpu; enum Method { - FGD_STAT, MOG, MOG2, - GMG + GMG, + FGD_STAT }; int main(int argc, const char** argv) @@ -29,7 +29,7 @@ int main(int argc, const char** argv) cv::CommandLineParser cmd(argc, argv, "{ c camera | | use camera }" "{ f file | 768x576.avi | input video file }" - "{ m method | mog | method (fgd, mog, mog2, gmg) }" + "{ m method | mog | method (mog, mog2, gmg, fgd) }" "{ h help | | print help message }"); if (cmd.has("help") || !cmd.check()) @@ -43,18 +43,18 @@ int main(int argc, const char** argv) string file = cmd.get("file"); string method = cmd.get("method"); - if (method != "fgd" - && method != "mog" + if (method != "mog" && method != "mog2" - && method != "gmg") + && method != "gmg" + && method != "fgd") { cerr << "Incorrect method" << endl; return -1; } - Method m = method == "fgd" ? FGD_STAT : - method == "mog" ? MOG : + Method m = method == "mog" ? MOG : method == "mog2" ? MOG2 : + method == "fgd" ? FGD_STAT : GMG; VideoCapture cap; @@ -75,11 +75,10 @@ int main(int argc, const char** argv) GpuMat d_frame(frame); + Ptr mog = gpu::createBackgroundSubtractorMOG(); + Ptr mog2 = gpu::createBackgroundSubtractorMOG2(); + Ptr gmg = gpu::createBackgroundSubtractorGMG(40); FGDStatModel fgd_stat; - cv::Ptr mog = cv::gpu::createBackgroundSubtractorMOG(); - cv::Ptr mog2 = cv::gpu::createBackgroundSubtractorMOG2(); - GMG_GPU gmg; - gmg.numInitializationFrames = 40; GpuMat d_fgmask; GpuMat d_fgimg; @@ -91,10 +90,6 @@ int main(int argc, const char** argv) switch (m) { - case FGD_STAT: - fgd_stat.create(d_frame); - break; - case MOG: mog->apply(d_frame, d_fgmask, 0.01); break; @@ -104,7 +99,11 @@ int main(int argc, const char** argv) break; case GMG: - gmg.initialize(d_frame.size()); + gmg->apply(d_frame, d_fgmask); + break; + + case FGD_STAT: + fgd_stat.create(d_frame); break; } @@ -128,12 +127,6 @@ int main(int argc, const char** argv) //update the model switch (m) { - case FGD_STAT: - fgd_stat.update(d_frame); - d_fgmask = fgd_stat.foreground; - d_bgimg = fgd_stat.background; - break; - case MOG: mog->apply(d_frame, d_fgmask, 0.01); mog->getBackgroundImage(d_bgimg); @@ -145,7 +138,13 @@ int main(int argc, const char** argv) break; case GMG: - gmg(d_frame, d_fgmask); + gmg->apply(d_frame, d_fgmask); + break; + + case FGD_STAT: + fgd_stat.update(d_frame); + d_fgmask = fgd_stat.foreground; + d_bgimg = fgd_stat.background; break; }