From 984583981c783a95a504b1b1e9390082d5cac662 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 22 Mar 2012 07:52:17 +0000 Subject: [PATCH] Split stabilizer into OnePassStabilizer and TwoPassStabilizer --- .../include/opencv2/videostab/deblurring.hpp | 20 +- .../opencv2/videostab/global_motion.hpp | 2 + .../include/opencv2/videostab/inpainting.hpp | 36 +- .../opencv2/videostab/motion_stabilizing.hpp | 37 +- .../include/opencv2/videostab/stabilizer.hpp | 113 ++++-- modules/videostab/src/global_motion.cpp | 12 +- modules/videostab/src/inpainting.cpp | 18 +- modules/videostab/src/motion_stabilizing.cpp | 22 +- modules/videostab/src/precomp.hpp | 12 +- modules/videostab/src/stabilizer.cpp | 370 +++++++++++------- samples/cpp/videostab.cpp | 58 ++- 11 files changed, 454 insertions(+), 246 deletions(-) diff --git a/modules/videostab/include/opencv2/videostab/deblurring.hpp b/modules/videostab/include/opencv2/videostab/deblurring.hpp index 2e881278af..a61f9ce618 100644 --- a/modules/videostab/include/opencv2/videostab/deblurring.hpp +++ b/modules/videostab/include/opencv2/videostab/deblurring.hpp @@ -53,24 +53,26 @@ namespace videostab CV_EXPORTS float calcBlurriness(const Mat &frame); -class CV_EXPORTS IDeblurer +class CV_EXPORTS DeblurerBase { public: - IDeblurer() : radius_(0), frames_(0), motions_(0) {} + DeblurerBase() : radius_(0), frames_(0), motions_(0) {} - virtual ~IDeblurer() {} + virtual ~DeblurerBase() {} virtual void setRadius(int val) { radius_ = val; } - int radius() const { return radius_; } + virtual int radius() const { return radius_; } virtual void setFrames(const std::vector &val) { frames_ = &val; } - const std::vector& frames() const { return *frames_; } + virtual const std::vector& frames() const { return *frames_; } virtual void setMotions(const std::vector &val) { motions_ = &val; } - const std::vector& motions() const { return *motions_; } + virtual const std::vector& motions() const { return *motions_; } virtual void setBlurrinessRates(const std::vector &val) { blurrinessRates_ = &val; } - const std::vector& blurrinessRates() const { return *blurrinessRates_; } + virtual const std::vector& blurrinessRates() const { return *blurrinessRates_; } + + virtual void update() {} virtual void deblur(int idx, Mat &frame) = 0; @@ -81,13 +83,13 @@ protected: const std::vector *blurrinessRates_; }; -class CV_EXPORTS NullDeblurer : public IDeblurer +class CV_EXPORTS NullDeblurer : public DeblurerBase { public: virtual void deblur(int /*idx*/, Mat &/*frame*/) {} }; -class CV_EXPORTS WeightingDeblurer : public IDeblurer +class CV_EXPORTS WeightingDeblurer : public DeblurerBase { public: WeightingDeblurer(); diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index e046bd0f4f..d62456df7a 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -129,6 +129,8 @@ private: float minInlierRatio_; }; +CV_EXPORTS Mat getMotion(int from, int to, const Mat *motions, int size); + CV_EXPORTS Mat getMotion(int from, int to, const std::vector &motions); } // namespace videostab diff --git a/modules/videostab/include/opencv2/videostab/inpainting.hpp b/modules/videostab/include/opencv2/videostab/inpainting.hpp index b973ba1388..7d3214265a 100644 --- a/modules/videostab/include/opencv2/videostab/inpainting.hpp +++ b/modules/videostab/include/opencv2/videostab/inpainting.hpp @@ -54,29 +54,31 @@ namespace cv namespace videostab { -class CV_EXPORTS IInpainter +class CV_EXPORTS InpainterBase { public: - IInpainter() + InpainterBase() : radius_(0), frames_(0), motions_(0), stabilizedFrames_(0), stabilizationMotions_(0) {} - virtual ~IInpainter() {} + virtual ~InpainterBase() {} virtual void setRadius(int val) { radius_ = val; } - int radius() const { return radius_; } + virtual int radius() const { return radius_; } virtual void setFrames(const std::vector &val) { frames_ = &val; } - const std::vector& frames() const { return *frames_; } + virtual const std::vector& frames() const { return *frames_; } virtual void setMotions(const std::vector &val) { motions_ = &val; } - const std::vector& motions() const { return *motions_; } + virtual const std::vector& motions() const { return *motions_; } virtual void setStabilizedFrames(const std::vector &val) { stabilizedFrames_ = &val; } - const std::vector& stabilizedFrames() const { return *stabilizedFrames_; } + virtual const std::vector& stabilizedFrames() const { return *stabilizedFrames_; } virtual void setStabilizationMotions(const std::vector &val) { stabilizationMotions_ = &val; } - const std::vector& stabilizationMotions() const { return *stabilizationMotions_; } + virtual const std::vector& stabilizationMotions() const { return *stabilizationMotions_; } + + virtual void update() {} virtual void inpaint(int idx, Mat &frame, Mat &mask) = 0; @@ -88,16 +90,16 @@ protected: const std::vector *stabilizationMotions_; }; -class CV_EXPORTS NullInpainter : public IInpainter +class CV_EXPORTS NullInpainter : public InpainterBase { public: virtual void inpaint(int /*idx*/, Mat &/*frame*/, Mat &/*mask*/) {} }; -class CV_EXPORTS InpaintingPipeline : public IInpainter +class CV_EXPORTS InpaintingPipeline : public InpainterBase { public: - void pushBack(Ptr inpainter) { inpainters_.push_back(inpainter); } + void pushBack(Ptr inpainter) { inpainters_.push_back(inpainter); } bool empty() const { return inpainters_.empty(); } virtual void setRadius(int val); @@ -106,13 +108,15 @@ public: virtual void setStabilizedFrames(const std::vector &val); virtual void setStabilizationMotions(const std::vector &val); + virtual void update(); + virtual void inpaint(int idx, Mat &frame, Mat &mask); private: - std::vector > inpainters_; + std::vector > inpainters_; }; -class CV_EXPORTS ConsistentMosaicInpainter : public IInpainter +class CV_EXPORTS ConsistentMosaicInpainter : public InpainterBase { public: ConsistentMosaicInpainter(); @@ -126,7 +130,7 @@ private: float stdevThresh_; }; -class CV_EXPORTS MotionInpainter : public IInpainter +class CV_EXPORTS MotionInpainter : public InpainterBase { public: MotionInpainter(); @@ -159,7 +163,7 @@ private: Mat_ flowMask_; }; -class CV_EXPORTS ColorAverageInpainter : public IInpainter +class CV_EXPORTS ColorAverageInpainter : public InpainterBase { public: virtual void inpaint(int idx, Mat &frame, Mat &mask); @@ -168,7 +172,7 @@ private: FastMarchingMethod fmm_; }; -class CV_EXPORTS ColorInpainter : public IInpainter +class CV_EXPORTS ColorInpainter : public InpainterBase { public: ColorInpainter(int method = INPAINT_TELEA, double radius = 2.) diff --git a/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp b/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp index f163a9a906..f811aa54ad 100644 --- a/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp +++ b/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp @@ -51,23 +51,44 @@ namespace cv namespace videostab { -class CV_EXPORTS IMotionFilter +class CV_EXPORTS IMotionStabilizer { public: - virtual ~IMotionFilter() {} - virtual int radius() const = 0; - virtual Mat apply(int index, std::vector &Ms) const = 0; + virtual void stabilize(const Mat *motions, int size, Mat *stabilizationMotions) const = 0; }; -class CV_EXPORTS GaussianMotionFilter : public IMotionFilter +class CV_EXPORTS MotionFilterBase : public IMotionStabilizer { public: - GaussianMotionFilter(int radius, float stdev); + MotionFilterBase() : radius_(0) {} + virtual ~MotionFilterBase() {} + + virtual void setRadius(int val) { radius_ = val; } virtual int radius() const { return radius_; } - virtual Mat apply(int idx, std::vector &motions) const; -private: + virtual void update() {} + + virtual Mat stabilize(int index, const Mat *motions, int size) const = 0; + virtual void stabilize(const Mat *motions, int size, Mat *stabilizationMotions) const; + +protected: int radius_; +}; + +class CV_EXPORTS GaussianMotionFilter : public MotionFilterBase +{ +public: + GaussianMotionFilter() : stdev_(-1.f) {} + + void setStdev(float val) { stdev_ = val; } + float stdev() const { return stdev_; } + + virtual void update(); + + virtual Mat stabilize(int index, const Mat *motions, int size) const; + +private: + float stdev_; std::vector weight_; }; diff --git a/modules/videostab/include/opencv2/videostab/stabilizer.hpp b/modules/videostab/include/opencv2/videostab/stabilizer.hpp index 99906312d8..261dd8b63c 100644 --- a/modules/videostab/include/opencv2/videostab/stabilizer.hpp +++ b/modules/videostab/include/opencv2/videostab/stabilizer.hpp @@ -58,79 +58,126 @@ namespace cv namespace videostab { -class CV_EXPORTS Stabilizer : public IFrameSource +class CV_EXPORTS StabilizerBase { public: - Stabilizer(); + virtual ~StabilizerBase() {} void setLog(Ptr log) { log_ = log; } Ptr log() const { return log_; } - void setFrameSource(Ptr val) { frameSource_ = val; reset(); } + void setRadius(int val) { radius_ = val; } + int radius() const { return radius_; } + + void setFrameSource(Ptr val) { frameSource_ = val; } Ptr frameSource() const { return frameSource_; } void setMotionEstimator(Ptr val) { motionEstimator_ = val; } Ptr motionEstimator() const { return motionEstimator_; } - void setMotionFilter(Ptr val) { motionFilter_ = val; reset(); } - Ptr motionFilter() const { return motionFilter_; } - - void setDeblurer(Ptr val) { deblurer_ = val; reset(); } - Ptr deblurrer() const { return deblurer_; } - - void setEstimateTrimRatio(bool val) { mustEstimateTrimRatio_ = val; reset(); } - bool mustEstimateTrimRatio() const { return mustEstimateTrimRatio_; } + void setDeblurer(Ptr val) { deblurer_ = val; } + Ptr deblurrer() const { return deblurer_; } - void setTrimRatio(float val) { trimRatio_ = val; reset(); } + void setTrimRatio(float val) { trimRatio_ = val; } float trimRatio() const { return trimRatio_; } - void setInclusionConstraint(bool val) { inclusionConstraint_ = val; } - bool inclusionConstraint() const { return inclusionConstraint_; } + void setCorrectionForInclusion(bool val) { doCorrectionForInclusion_ = val; } + bool doCorrectionForInclusion() const { return doCorrectionForInclusion_; } void setBorderMode(int val) { borderMode_ = val; } int borderMode() const { return borderMode_; } - void setInpainter(Ptr val) { inpainter_ = val; reset(); } - Ptr inpainter() const { return inpainter_; } + void setInpainter(Ptr val) { inpainter_ = val; } + Ptr inpainter() const { return inpainter_; } - virtual void reset(); - virtual Mat nextFrame(); +protected: + StabilizerBase(); -private: - void estimateMotionsAndTrimRatio(); - void processFirstFrame(Mat &frame); - bool processNextFrame(); - void stabilizeFrame(int idx); + void setUp(int cacheSize, const Mat &frame); + Mat nextStabilizedFrame(); + bool doOneIteration(); + void stabilizeFrame(const Mat &stabilizationMotion); + virtual void setUp(Mat &firstFrame) = 0; + virtual void stabilizeFrame() = 0; + virtual void estimateMotion() = 0; + + Ptr log_; Ptr frameSource_; Ptr motionEstimator_; - Ptr motionFilter_; - Ptr deblurer_; - Ptr inpainter_; - bool mustEstimateTrimRatio_; + Ptr deblurer_; + Ptr inpainter_; + int radius_; float trimRatio_; - bool inclusionConstraint_; - int borderMode_; - Ptr log_; + bool doCorrectionForInclusion_; + int borderMode_; Size frameSize_; Mat frameMask_; - int radius_; int curPos_; int curStabilizedPos_; - bool auxPassWasDone_; bool doDeblurring_; Mat preProcessedFrame_; bool doInpainting_; Mat inpaintingMask_; std::vector frames_; - std::vector motions_; // motions_[i] is the motion from i to i+1 frame + std::vector motions_; // motions_[i] is the motion from i-th to i+1-th frame std::vector blurrinessRates_; std::vector stabilizedFrames_; std::vector stabilizedMasks_; std::vector stabilizationMotions_; }; +class CV_EXPORTS OnePassStabilizer : public StabilizerBase, public IFrameSource +{ +public: + OnePassStabilizer(); + + void setMotionFilter(Ptr val) { motionFilter_ = val; } + Ptr motionFilter() const { return motionFilter_; } + + virtual void reset() { resetImpl(); } + virtual Mat nextFrame() { return nextStabilizedFrame(); } + +private: + void resetImpl(); + + virtual void setUp(Mat &firstFrame); + virtual void estimateMotion(); + virtual void stabilizeFrame(); + + Ptr motionFilter_; +}; + +class CV_EXPORTS TwoPassStabilizer : public StabilizerBase, public IFrameSource +{ +public: + TwoPassStabilizer(); + + void setMotionStabilizer(Ptr val) { motionStabilizer_ = val; } + Ptr motionStabilizer() const { return motionStabilizer_; } + + void setEstimateTrimRatio(bool val) { mustEstTrimRatio_ = val; } + bool mustEstimateTrimaRatio() const { return mustEstTrimRatio_; } + + virtual void reset() { resetImpl(); } + virtual Mat nextFrame(); + +private: + void resetImpl(); + void runPrePassIfNecessary(); + + virtual void setUp(Mat &firstFrame); + virtual void estimateMotion() { /* do nothing as motion was estimation in pre-pass */ } + virtual void stabilizeFrame(); + + Ptr motionStabilizer_; + bool mustEstTrimRatio_; + + int frameCount_; + bool isPrePassDone_; +}; + } // namespace videostab } // namespace cv diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index d015d1b1b3..94292b3289 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -296,22 +296,28 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1) } -Mat getMotion(int from, int to, const vector &motions) +Mat getMotion(int from, int to, const Mat *motions, int size) { Mat M = Mat::eye(3, 3, CV_32F); if (to > from) { for (int i = from; i < to; ++i) - M = at(i, motions) * M; + M = at(i, motions, size) * M; } else if (from > to) { for (int i = to; i < from; ++i) - M = at(i, motions) * M; + M = at(i, motions, size) * M; M = M.inv(); } return M; } + +Mat getMotion(int from, int to, const vector &motions) +{ + return getMotion(from, to, &motions[0], motions.size()); +} + } // namespace videostab } // namespace cv diff --git a/modules/videostab/src/inpainting.cpp b/modules/videostab/src/inpainting.cpp index 479e5af2ed..805458dbf7 100644 --- a/modules/videostab/src/inpainting.cpp +++ b/modules/videostab/src/inpainting.cpp @@ -57,7 +57,7 @@ void InpaintingPipeline::setRadius(int val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setRadius(val); - IInpainter::setRadius(val); + InpainterBase::setRadius(val); } @@ -65,7 +65,7 @@ void InpaintingPipeline::setFrames(const vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setFrames(val); - IInpainter::setFrames(val); + InpainterBase::setFrames(val); } @@ -73,7 +73,7 @@ void InpaintingPipeline::setMotions(const vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setMotions(val); - IInpainter::setMotions(val); + InpainterBase::setMotions(val); } @@ -81,7 +81,7 @@ void InpaintingPipeline::setStabilizedFrames(const vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setStabilizedFrames(val); - IInpainter::setStabilizedFrames(val); + InpainterBase::setStabilizedFrames(val); } @@ -89,7 +89,15 @@ void InpaintingPipeline::setStabilizationMotions(const vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setStabilizationMotions(val); - IInpainter::setStabilizationMotions(val); + InpainterBase::setStabilizationMotions(val); +} + + +void InpaintingPipeline::update() +{ + for (size_t i = 0; i < inpainters_.size(); ++i) + inpainters_[i]->update(); + InpainterBase::update(); } diff --git a/modules/videostab/src/motion_stabilizing.cpp b/modules/videostab/src/motion_stabilizing.cpp index 047f627ab8..912f5675f1 100644 --- a/modules/videostab/src/motion_stabilizing.cpp +++ b/modules/videostab/src/motion_stabilizing.cpp @@ -51,26 +51,34 @@ namespace cv namespace videostab { -GaussianMotionFilter::GaussianMotionFilter(int radius, float stdev) : radius_(radius) +void MotionFilterBase::stabilize(const Mat *motions, int size, Mat *stabilizationMotions) const { + for (int i = 0; i < size; ++i) + stabilizationMotions[i] = stabilize(i, motions, size); +} + + +void GaussianMotionFilter::update() +{ + float sigma = stdev_ > 0.f ? stdev_ : sqrt(static_cast(radius_)); float sum = 0; weight_.resize(2*radius_ + 1); for (int i = -radius_; i <= radius_; ++i) - sum += weight_[radius_ + i] = std::exp(-i*i/(stdev*stdev)); + sum += weight_[radius_ + i] = std::exp(-i*i/(sigma*sigma)); for (int i = -radius_; i <= radius_; ++i) weight_[radius_ + i] /= sum; } -Mat GaussianMotionFilter::apply(int idx, vector &motions) const +Mat GaussianMotionFilter::stabilize(int index, const Mat *motions, int size) const { - const Mat &cur = at(idx, motions); + const Mat &cur = at(index, motions, size); Mat res = Mat::zeros(cur.size(), cur.type()); float sum = 0.f; - for (int i = std::max(idx - radius_, 0); i <= idx + radius_; ++i) + for (int i = std::max(index - radius_, 0); i <= index + radius_; ++i) { - res += weight_[radius_ + i - idx] * getMotion(idx, i, motions); - sum += weight_[radius_ + i - idx]; + res += weight_[radius_ + i - index] * getMotion(index, i, motions, size); + sum += weight_[radius_ + i - index]; } return res / sum; } diff --git a/modules/videostab/src/precomp.hpp b/modules/videostab/src/precomp.hpp index 844fbef6c4..eb322d128e 100644 --- a/modules/videostab/src/precomp.hpp +++ b/modules/videostab/src/precomp.hpp @@ -62,9 +62,19 @@ inline float intensity(const cv::Point3_ &bgr) return 0.3f*bgr.x + 0.59f*bgr.y + 0.11f*bgr.z; } +template inline T& at(int index, const T *items, int size) +{ + return items[cv::borderInterpolate(index, size, cv::BORDER_WRAP)]; +} + +template inline const T& at(int index, const T *items, int size) +{ + return items[cv::borderInterpolate(index, size, cv::BORDER_WRAP)]; +} + template inline T& at(int index, std::vector &items) { - return items[cv::borderInterpolate(index, items.size(), cv::BORDER_WRAP)]; + return at(index, &items[0], items.size()); } template inline const T& at(int index, const std::vector &items) diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index ac7fd9cda5..5b1714715a 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -50,130 +50,23 @@ namespace cv namespace videostab { -Stabilizer::Stabilizer() +StabilizerBase::StabilizerBase() { + setLog(new NullLog()); setFrameSource(new NullFrameSource()); setMotionEstimator(new PyrLkRobustMotionEstimator()); - setMotionFilter(new GaussianMotionFilter(15, sqrt(15.f))); setDeblurer(new NullDeblurer()); setInpainter(new NullInpainter()); - setEstimateTrimRatio(true); + setRadius(15); setTrimRatio(0); - setInclusionConstraint(false); + setCorrectionForInclusion(false); setBorderMode(BORDER_REPLICATE); - setLog(new NullLog()); -} - - -void Stabilizer::reset() -{ - radius_ = 0; - curPos_ = -1; - curStabilizedPos_ = -1; - auxPassWasDone_ = false; - frames_.clear(); - motions_.clear(); - stabilizedFrames_.clear(); - stabilizationMotions_.clear(); - doDeblurring_ = false; - doInpainting_ = false; -} - - -Mat Stabilizer::nextFrame() -{ - if (mustEstimateTrimRatio_ && !auxPassWasDone_) - { - estimateMotionsAndTrimRatio(); - auxPassWasDone_ = true; - frameSource_->reset(); - } - - if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1) - return Mat(); // we've processed all frames already - - bool processed; - do { - processed = processNextFrame(); - } while (processed && curStabilizedPos_ == -1); - - if (curStabilizedPos_ == -1) - return Mat(); // frame source is empty - - const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_); - int dx = static_cast(floor(trimRatio_ * stabilizedFrame.cols)); - int dy = static_cast(floor(trimRatio_ * stabilizedFrame.rows)); - return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy)); -} - - -void Stabilizer::estimateMotionsAndTrimRatio() -{ - log_->print("estimating motions and trim ratio"); - - Size size; - Mat prevFrame, frame; - int frameCount = 0; - - while (!(frame = frameSource_->nextFrame()).empty()) - { - if (frameCount > 0) - motions_.push_back(motionEstimator_->estimate(prevFrame, frame)); - else - size = frame.size(); - prevFrame = frame; - frameCount++; - - log_->print("."); - } - - radius_ = motionFilter_->radius(); - for (int i = 0; i < radius_; ++i) - motions_.push_back(Mat::eye(3, 3, CV_32F)); - log_->print("\n"); - - trimRatio_ = 0; - for (int i = 0; i < frameCount; ++i) - { - Mat S = motionFilter_->apply(i, motions_); - trimRatio_ = std::max(trimRatio_, estimateOptimalTrimRatio(S, size)); - stabilizationMotions_.push_back(S); - } - - log_->print("estimated trim ratio: %f\n", static_cast(trimRatio_)); } -void Stabilizer::processFirstFrame(Mat &frame) +void StabilizerBase::setUp(int cacheSize, const Mat &frame) { - log_->print("processing frames"); - - frameSize_ = frame.size(); - frameMask_.create(frameSize_, CV_8U); - frameMask_.setTo(255); - - radius_ = motionFilter_->radius(); - int cacheSize = 2*radius_ + 1; - - frames_.resize(cacheSize); - stabilizedFrames_.resize(cacheSize); - stabilizedMasks_.resize(cacheSize); - - if (!auxPassWasDone_) - { - motions_.resize(cacheSize); - stabilizationMotions_.resize(cacheSize); - } - - for (int i = -radius_; i < 0; ++i) - { - at(i, motions_) = Mat::eye(3, 3, CV_32F); - at(i, frames_) = frame; - } - - at(0, frames_) = frame; - - IInpainter *inpainter = static_cast(inpainter_); + InpainterBase *inpainter = static_cast(inpainter_); doInpainting_ = dynamic_cast(inpainter) == 0; if (doInpainting_) { @@ -182,9 +75,10 @@ void Stabilizer::processFirstFrame(Mat &frame) inpainter_->setMotions(motions_); inpainter_->setStabilizedFrames(stabilizedFrames_); inpainter_->setStabilizationMotions(stabilizationMotions_); + inpainter_->update(); } - IDeblurer *deblurer = static_cast(deblurer_); + DeblurerBase *deblurer = static_cast(deblurer_); doDeblurring_ = dynamic_cast(deblurer) == 0; if (doDeblurring_) { @@ -196,11 +90,33 @@ void Stabilizer::processFirstFrame(Mat &frame) deblurer_->setFrames(frames_); deblurer_->setMotions(motions_); deblurer_->setBlurrinessRates(blurrinessRates_); + deblurer_->update(); } + + log_->print("processing frames"); +} + + +Mat StabilizerBase::nextStabilizedFrame() +{ + if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1) + return Mat(); // we've processed all frames already + + bool processed; + do processed = doOneIteration(); + while (processed && curStabilizedPos_ == -1); + + if (curStabilizedPos_ == -1) + return Mat(); // frame source is empty + + const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_); + int dx = static_cast(floor(trimRatio_ * stabilizedFrame.cols)); + int dy = static_cast(floor(trimRatio_ * stabilizedFrame.rows)); + return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy)); } -bool Stabilizer::processNextFrame() +bool StabilizerBase::doOneIteration() { Mat frame = frameSource_->nextFrame(); if (!frame.empty()) @@ -214,21 +130,16 @@ bool Stabilizer::processNextFrame() if (doDeblurring_) at(curPos_, blurrinessRates_) = calcBlurriness(frame); - if (!auxPassWasDone_) - { - Mat motionPrevToCur = motionEstimator_->estimate( - at(curPos_ - 1, frames_), at(curPos_, frames_)); - at(curPos_ - 1, motions_) = motionPrevToCur; - } + estimateMotion(); if (curPos_ >= radius_) { curStabilizedPos_ = curPos_ - radius_; - stabilizeFrame(curStabilizedPos_); + stabilizeFrame(); } } else - processFirstFrame(frame); + setUp(frame); log_->print("."); return true; @@ -239,7 +150,7 @@ bool Stabilizer::processNextFrame() curStabilizedPos_++; at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_); at(curStabilizedPos_ + radius_ - 1, motions_) = at(curPos_ - 1, motions_); - stabilizeFrame(curStabilizedPos_); + stabilizeFrame(); log_->print("."); return true; @@ -249,42 +160,215 @@ bool Stabilizer::processNextFrame() } -void Stabilizer::stabilizeFrame(int idx) +void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion) { - Mat stabMotion; - if (!auxPassWasDone_) - stabMotion = motionFilter_->apply(idx, motions_); + Mat stabilizationMotion_; + if (doCorrectionForInclusion_) + stabilizationMotion_ = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); else - stabMotion = at(idx, stabilizationMotions_); - - if (inclusionConstraint_ && !mustEstimateTrimRatio_) - stabMotion = ensureInclusionConstraint(stabMotion, frameSize_, trimRatio_); + stabilizationMotion_ = stabilizationMotion.clone(); - at(idx, stabilizationMotions_) = stabMotion; + at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion_; if (doDeblurring_) { - at(idx, frames_).copyTo(preProcessedFrame_); - deblurer_->deblur(idx, preProcessedFrame_); + at(curStabilizedPos_, frames_).copyTo(preProcessedFrame_); + deblurer_->deblur(curStabilizedPos_, preProcessedFrame_); } else - preProcessedFrame_ = at(idx, frames_); + preProcessedFrame_ = at(curStabilizedPos_, frames_); // apply stabilization transformation warpAffine( - preProcessedFrame_, at(idx, stabilizedFrames_), stabMotion(Rect(0,0,3,2)), - frameSize_, INTER_LINEAR, borderMode_); + preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), + stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); if (doInpainting_) { warpAffine( - frameMask_, at(idx, stabilizedMasks_), stabMotion(Rect(0,0,3,2)), frameSize_, - INTER_NEAREST); - erode(at(idx, stabilizedMasks_), at(idx, stabilizedMasks_), Mat()); - at(idx, stabilizedMasks_).copyTo(inpaintingMask_); - inpainter_->inpaint(idx, at(idx, stabilizedFrames_), inpaintingMask_); + frameMask_, at(curStabilizedPos_, stabilizedMasks_), + stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); + + erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_), + Mat()); + + at(curStabilizedPos_, stabilizedMasks_).copyTo(inpaintingMask_); + + inpainter_->inpaint( + curStabilizedPos_, at(curStabilizedPos_, stabilizedFrames_), inpaintingMask_); } } + +OnePassStabilizer::OnePassStabilizer() +{ + setMotionFilter(new GaussianMotionFilter()); + resetImpl(); +} + + +void OnePassStabilizer::resetImpl() +{ + curPos_ = -1; + curStabilizedPos_ = -1; + frames_.clear(); + motions_.clear(); + stabilizedFrames_.clear(); + stabilizationMotions_.clear(); + doDeblurring_ = false; + doInpainting_ = false; +} + + +void OnePassStabilizer::setUp(Mat &firstFrame) +{ + frameSize_ = firstFrame.size(); + frameMask_.create(frameSize_, CV_8U); + frameMask_.setTo(255); + + int cacheSize = 2*radius_ + 1; + + frames_.resize(cacheSize); + stabilizedFrames_.resize(cacheSize); + stabilizedMasks_.resize(cacheSize); + motions_.resize(cacheSize); + stabilizationMotions_.resize(cacheSize); + + for (int i = -radius_; i < 0; ++i) + { + at(i, motions_) = Mat::eye(3, 3, CV_32F); + at(i, frames_) = firstFrame; + } + + at(0, frames_) = firstFrame; + + motionFilter_->setRadius(radius_); + motionFilter_->update(); + + StabilizerBase::setUp(cacheSize, firstFrame); +} + + +void OnePassStabilizer::estimateMotion() +{ + at(curPos_ - 1, motions_) = motionEstimator_->estimate( + at(curPos_ - 1, frames_), at(curPos_, frames_)); +} + + +void OnePassStabilizer::stabilizeFrame() +{ + Mat stabilizationMotion = motionFilter_->stabilize(curStabilizedPos_, &motions_[0], motions_.size()); + StabilizerBase::stabilizeFrame(stabilizationMotion); +} + + +TwoPassStabilizer::TwoPassStabilizer() +{ + setMotionStabilizer(new GaussianMotionFilter()); + setEstimateTrimRatio(true); + resetImpl(); +} + + +Mat TwoPassStabilizer::nextFrame() +{ + runPrePassIfNecessary(); + return StabilizerBase::nextStabilizedFrame(); +} + + +void TwoPassStabilizer::resetImpl() +{ + isPrePassDone_ = false; + frameCount_ = 0; + curPos_ = -1; + curStabilizedPos_ = -1; + frames_.clear(); + motions_.clear(); + stabilizedFrames_.clear(); + stabilizationMotions_.clear(); + doDeblurring_ = false; + doInpainting_ = false; +} + + +void TwoPassStabilizer::runPrePassIfNecessary() +{ + if (!isPrePassDone_) + { + log_->print("first pass: estimating motions"); + + Mat prevFrame, frame; + + while (!(frame = frameSource_->nextFrame()).empty()) + { + if (frameCount_ > 0) + motions_.push_back(motionEstimator_->estimate(prevFrame, frame)); + else + { + frameSize_ = frame.size(); + frameMask_.create(frameSize_, CV_8U); + frameMask_.setTo(255); + } + + prevFrame = frame; + frameCount_++; + + log_->print("."); + } + + for (int i = 0; i < radius_; ++i) + motions_.push_back(Mat::eye(3, 3, CV_32F)); + log_->print("\n"); + + IMotionStabilizer *motionStabilizer = static_cast(motionStabilizer_); + MotionFilterBase *motionFilterBase = dynamic_cast(motionStabilizer); + if (motionFilterBase) + { + motionFilterBase->setRadius(radius_); + motionFilterBase->update(); + } + + stabilizationMotions_.resize(frameCount_); + motionStabilizer_->stabilize(&motions_[0], frameCount_, &stabilizationMotions_[0]); + + if (mustEstTrimRatio_) + { + trimRatio_ = 0; + for (int i = 0; i < frameCount_; ++i) + { + Mat S = stabilizationMotions_[i]; + trimRatio_ = std::max(trimRatio_, estimateOptimalTrimRatio(S, frameSize_)); + } + log_->print("estimated trim ratio: %f\n", static_cast(trimRatio_)); + } + + isPrePassDone_ = true; + frameSource_->reset(); + } +} + + +void TwoPassStabilizer::setUp(Mat &firstFrame) +{ + int cacheSize = 2*radius_ + 1; + + frames_.resize(cacheSize); + stabilizedFrames_.resize(cacheSize); + stabilizedMasks_.resize(cacheSize); + + for (int i = -radius_; i <= 0; ++i) + at(i, frames_) = firstFrame; + + StabilizerBase::setUp(cacheSize, firstFrame); +} + + +void TwoPassStabilizer::stabilizeFrame() +{ + StabilizerBase::stabilizeFrame(stabilizationMotions_[curStabilizedPos_]); +} + } // namespace videostab } // namespace cv diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index 807438ff6a..b5f62c08a5 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -12,7 +12,7 @@ using namespace std; using namespace cv; using namespace cv::videostab; -Ptr stabilizer; +Ptr stabilizedFrames; double outputFps; string outputPath; bool quietMode; @@ -25,7 +25,7 @@ void run() VideoWriter writer; Mat stabilizedFrame; - while (!(stabilizedFrame = stabilizer->nextFrame()).empty()) + while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty()) { if (!outputPath.empty()) { @@ -87,7 +87,7 @@ void printHelp() " Do color inpainting. The defailt is no.\n" " --color-inpaint-radius=\n" " Set color inpainting radius (for ns and telea options only).\n\n" - " -o, --output=\n" + " -o, --output=(no|)\n" " Set output file path explicitely. The default is stabilized.avi.\n" " --fps=\n" " Set output video FPS explicitely. By default the source FPS is used.\n" @@ -134,9 +134,35 @@ int main(int argc, const char **argv) { printHelp(); return 0; + } + + StabilizerBase *stabilizer; + GaussianMotionFilter *motionFilter = 0; + + if (!cmd.get("stdev").empty()) + { + motionFilter = new GaussianMotionFilter(); + motionFilter->setStdev(cmd.get("stdev")); } - stabilizer = new Stabilizer(); + bool isTwoPass = cmd.get("est-trim") == "yes"; + + if (isTwoPass) + { + TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer(); + if (!cmd.get("est-trim").empty()) + twoPassStabilizer->setEstimateTrimRatio(cmd.get("est-trim") == "yes"); + if (motionFilter) + twoPassStabilizer->setMotionStabilizer(motionFilter); + stabilizer = twoPassStabilizer; + } + else + { + OnePassStabilizer *onePassStabilizer= new OnePassStabilizer(); + if (motionFilter) + onePassStabilizer->setMotionFilter(motionFilter); + stabilizer = onePassStabilizer; + } string inputPath = cmd.get("1"); if (inputPath.empty()) @@ -169,16 +195,8 @@ int main(int argc, const char **argv) stabilizer->setMotionEstimator(motionEstimator); - int smoothRadius = -1; - float smoothStdev = -1; if (!cmd.get("radius").empty()) - smoothRadius = cmd.get("radius"); - if (!cmd.get("stdev").empty()) - smoothStdev = cmd.get("stdev"); - if (smoothRadius > 0 && smoothStdev > 0) - stabilizer->setMotionFilter(new GaussianMotionFilter(smoothRadius, smoothStdev)); - else if (smoothRadius > 0 && smoothStdev < 0) - stabilizer->setMotionFilter(new GaussianMotionFilter(smoothRadius, sqrt(static_cast(smoothRadius)))); + stabilizer->setRadius(cmd.get("radius")); if (cmd.get("deblur") == "yes") { @@ -188,14 +206,11 @@ int main(int argc, const char **argv) stabilizer->setDeblurer(deblurer); } - if (!cmd.get("est-trim").empty()) - stabilizer->setEstimateTrimRatio(cmd.get("est-trim") == "yes"); - if (!cmd.get("trim-ratio").empty()) stabilizer->setTrimRatio(cmd.get("trim-ratio")); if (!cmd.get("incl-constr").empty()) - stabilizer->setInclusionConstraint(cmd.get("incl-constr") == "yes"); + stabilizer->setCorrectionForInclusion(cmd.get("incl-constr") == "yes"); if (cmd.get("border-mode") == "reflect") stabilizer->setBorderMode(BORDER_REFLECT); @@ -250,22 +265,23 @@ int main(int argc, const char **argv) stabilizer->setLog(new LogToStdout()); - outputPath = cmd.get("output"); + outputPath = cmd.get("output") != "no" ? cmd.get("output") : ""; if (!cmd.get("fps").empty()) outputFps = cmd.get("fps"); quietMode = cmd.get("quiet"); - // run video processing + stabilizedFrames = dynamic_cast(stabilizer); + run(); } catch (const exception &e) { cout << "error: " << e.what() << endl; - stabilizer.release(); + stabilizedFrames.release(); return -1; } - stabilizer.release(); + stabilizedFrames.release(); return 0; }