diff --git a/modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp b/modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp index 274368861..07bcf6d9a 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp @@ -39,6 +39,7 @@ #ifdef __cplusplus #include +#include namespace cv { namespace ximgproc { @@ -63,15 +64,15 @@ public: @param filtered_disparity_map output disparity map. - @param ROI region of the disparity map to filter. - @param disparity_map_right optional argument, some implementations might also use the disparity map of the right view to compute confidence maps, for instance. + @param ROI region of the disparity map to filter. Optional, usually it should be set automatically. + @param right_view optional argument, some implementations might also use the right view of the original stereo-pair. */ - CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right = Mat(), InputArray right_view = Mat()) = 0; + CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right = Mat(), Rect ROI = Rect(), InputArray right_view = Mat()) = 0; }; /** @brief Disparity map filter based on Weighted Least Squares filter (in form of Fast Global Smoother that @@ -106,8 +107,7 @@ public: /** @see getLRCthresh */ CV_WRAP virtual void setLRCthresh(int _LRC_thresh) = 0; /** @brief DepthDiscontinuityRadius is a parameter used in confidence computation. It defines the size of - low-confidence regions around depth discontinuities. For typical window sizes used in stereo matching the - optimal value is around 5. + low-confidence regions around depth discontinuities. */ CV_WRAP virtual int getDepthDiscontinuityRadius() = 0; /** @see getDepthDiscontinuityRadius */ @@ -117,16 +117,36 @@ public: correct disparity values with a high degree of confidence). */ CV_WRAP virtual Mat getConfidenceMap() = 0; - + /** @brief Get the ROI used in the last filter call + */ + CV_WRAP virtual Rect getROI() = 0; }; -/** @brief Factory method, create instance of DisparityWLSFilter and execute the initialization routines. +/** @brief Convenience factory method that creates an instance of DisparityWLSFilter and sets up all the relevant +filter parameters automatically based on the matcher instance. Currently supports only StereoBM and StereoSGBM. + +@param matcher_left stereo matcher instance that will be used with the filter +*/ +CV_EXPORTS_W +Ptr createDisparityWLSFilter(Ptr matcher_left); + +/** @brief Convenience method to set up the matcher for computing the right-view disparity map +that is required in case of filtering with confidence. + +@param matcher_left main stereo matcher instance that will be used with the filter +*/ +CV_EXPORTS_W +Ptr createRightMatcher(Ptr matcher_left); + +/** @brief More generic factory method, create instance of DisparityWLSFilter and execute basic +initialization routines. When using this method you will need to set-up the ROI, matchers and +other parameters by yourself. @param use_confidence filtering with confidence requires two disparity maps (for the left and right views) and is approximately two times slower. However, quality is typically significantly better. */ CV_EXPORTS_W -Ptr createDisparityWLSFilter(bool use_confidence); +Ptr createDisparityWLSFilterGeneric(bool use_confidence); ////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////// diff --git a/modules/ximgproc/perf/perf_disparity_wls_filter.cpp b/modules/ximgproc/perf/perf_disparity_wls_filter.cpp index c4fbc1298..46d0ac3ee 100644 --- a/modules/ximgproc/perf/perf_disparity_wls_filter.cpp +++ b/modules/ximgproc/perf/perf_disparity_wls_filter.cpp @@ -85,8 +85,8 @@ PERF_TEST_P( DisparityWLSFilterPerfTest, perf, Combine(GuideTypes::all(), SrcTyp cv::setNumThreads(cv::getNumberOfCPUs()); TEST_CYCLE_N(10) { - Ptr wls_filter = createDisparityWLSFilter(use_conf); - wls_filter->filter(disp_left,guide,dst,ROI,disp_right); + Ptr wls_filter = createDisparityWLSFilterGeneric(use_conf); + wls_filter->filter(disp_left,guide,dst,disp_right,ROI); } SANITY_CHECK(dst); diff --git a/modules/ximgproc/samples/disparity_filtering.cpp b/modules/ximgproc/samples/disparity_filtering.cpp index 5ed66ec69..beef61d76 100644 --- a/modules/ximgproc/samples/disparity_filtering.cpp +++ b/modules/ximgproc/samples/disparity_filtering.cpp @@ -23,13 +23,13 @@ const String keys = "{algorithm |bm | stereo matching method (bm or sgbm) }" "{filter |wls_conf | used post-filtering (wls_conf or wls_no_conf) }" "{no-display | | don't display results }" - "{no-downscale | | prevent stereo matching on downscaled views }" + "{no-downscale | | force stereo matching on full-sized views to improve quality }" "{dst_conf_path |None | optional path to save the confidence map used in filtering }" "{vis_mult |1.0 | coefficient used to scale disparity map visualizations }" "{max_disparity |160 | parameter of stereo matching }" - "{window_size |19 | parameter of stereo matching }" + "{window_size |-1 | parameter of stereo matching }" "{wls_lambda |8000.0 | parameter of post-filtering }" - "{wls_sigma |1.0 | parameter of post-filtering }" + "{wls_sigma |1.5 | parameter of post-filtering }" ; int main(int argc, char** argv) @@ -54,17 +54,30 @@ int main(int argc, char** argv) bool no_display = parser.has("no-display"); bool no_downscale = parser.has("no-downscale"); int max_disp = parser.get("max_disparity"); - int wsize = parser.get("window_size"); double lambda = parser.get("wls_lambda"); double sigma = parser.get("wls_sigma"); double vis_mult = parser.get("vis_mult"); + int wsize; + if(parser.get("window_size")>=0) //user provided window_size value + wsize = parser.get("window_size"); + else + { + if(algo=="sgbm") + wsize = 3; //default window size for SGBM + else if(!no_downscale && algo=="bm" && filter=="wls_conf") + wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf) + else + wsize = 15; //default window size for BM on full-sized views + } + if (!parser.check()) { parser.printErrors(); return -1; } + //! [load_views] Mat left = imread(left_im ,IMREAD_COLOR); if ( left.empty() ) { @@ -78,6 +91,7 @@ int main(int argc, char** argv) cout<<"Cannot read image file: "< wls_filter; double matching_time, filtering_time; if(max_disp<=0 || max_disp%16!=0) { @@ -110,17 +125,19 @@ int main(int argc, char** argv) cout<<"Incorrect window_size value: it should be positive and odd"; return -1; } - if(filter=="wls_conf") + if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf) { if(!no_downscale) { - wsize = wsize/2; - if(wsize%2==0) wsize++; + // downscale the views to speed-up the matching stage, as we will need to compute both left + // and right disparity maps for confidence map computation + //! [downscale] max_disp/=2; if(max_disp%16!=0) max_disp += 16-(max_disp%16); resize(left ,left_for_matcher ,Size(),0.5,0.5); resize(right,right_for_matcher,Size(),0.5,0.5); + //! [downscale] } else { @@ -128,38 +145,31 @@ int main(int argc, char** argv) right_for_matcher = right.clone(); } - if(algo=="bm") { - Ptr left_matcher = StereoBM::create(max_disp,wsize); - left_matcher->setMinDisparity(0); - Ptr right_matcher = StereoBM::create(max_disp,wsize); - right_matcher->setMinDisparity(-max_disp+1); - left_matcher->setTextureThreshold(0); - left_matcher->setUniquenessRatio(0); - right_matcher->setTextureThreshold(0); - right_matcher->setUniquenessRatio(0); - cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY); + //! [matching] + Ptr left_matcher = StereoBM::create(max_disp,wsize); + wls_filter = createDisparityWLSFilter(left_matcher); + Ptr right_matcher = createRightMatcher(left_matcher); + + cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY); cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY); - ROI = computeROI(left_for_matcher.size(),left_matcher); matching_time = (double)getTickCount(); left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp); right_matcher->compute(right_for_matcher,left_for_matcher, right_disp); matching_time = ((double)getTickCount() - matching_time)/getTickFrequency(); + //! [matching] } else if(algo=="sgbm") { Ptr left_matcher = StereoSGBM::create(0,max_disp,wsize); - left_matcher->setMinDisparity(0); - Ptr right_matcher = StereoSGBM::create(-max_disp+1,max_disp,wsize); - left_matcher->setUniquenessRatio(0); - left_matcher->setDisp12MaxDiff(1000000); - left_matcher->setSpeckleWindowSize(0); - right_matcher->setUniquenessRatio(0); - right_matcher->setDisp12MaxDiff(1000000); - right_matcher->setSpeckleWindowSize(0); - ROI = computeROI(left_for_matcher.size(),left_matcher); + left_matcher->setP1(24*wsize*wsize); + left_matcher->setP2(96*wsize*wsize); + left_matcher->setPreFilterCap(63); + left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY); + wls_filter = createDisparityWLSFilter(left_matcher); + Ptr right_matcher = createRightMatcher(left_matcher); matching_time = (double)getTickCount(); left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp); @@ -172,14 +182,17 @@ int main(int argc, char** argv) return -1; } - Ptr wls_filter = createDisparityWLSFilter(true); + //! [filtering] wls_filter->setLambda(lambda); wls_filter->setSigmaColor(sigma); filtering_time = (double)getTickCount(); - wls_filter->filter(left_disp,left,filtered_disp,ROI,right_disp); + wls_filter->filter(left_disp,left,filtered_disp,right_disp); filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency(); + //! [filtering] conf_map = wls_filter->getConfidenceMap(); + // Get the ROI that was used in the last filter call: + ROI = wls_filter->getROI(); if(!no_downscale) { // upscale raw disparity and ROI back for a proper comparison: @@ -190,6 +203,9 @@ int main(int argc, char** argv) } else if(filter=="wls_no_conf") { + /* There is no convenience function for the case of filtering with no confidence, so we + will need to set the ROI and matcher parameters manually */ + left_for_matcher = left.clone(); right_for_matcher = right.clone(); @@ -201,6 +217,8 @@ int main(int argc, char** argv) cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY); cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY); ROI = computeROI(left_for_matcher.size(),matcher); + wls_filter = createDisparityWLSFilterGeneric(false); + wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize)); matching_time = (double)getTickCount(); matcher->compute(left_for_matcher,right_for_matcher,left_disp); @@ -212,7 +230,12 @@ int main(int argc, char** argv) matcher->setUniquenessRatio(0); matcher->setDisp12MaxDiff(1000000); matcher->setSpeckleWindowSize(0); + matcher->setP1(24*wsize*wsize); + matcher->setP2(96*wsize*wsize); + matcher->setMode(StereoSGBM::MODE_SGBM_3WAY); ROI = computeROI(left_for_matcher.size(),matcher); + wls_filter = createDisparityWLSFilterGeneric(false); + wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize)); matching_time = (double)getTickCount(); matcher->compute(left_for_matcher,right_for_matcher,left_disp); @@ -224,11 +247,10 @@ int main(int argc, char** argv) return -1; } - Ptr wls_filter = createDisparityWLSFilter(false); wls_filter->setLambda(lambda); wls_filter->setSigmaColor(sigma); filtering_time = (double)getTickCount(); - wls_filter->filter(left_disp,left,filtered_disp,ROI); + wls_filter->filter(left_disp,left,filtered_disp,Mat(),ROI); filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency(); } else @@ -292,6 +314,7 @@ int main(int argc, char** argv) imshow("ground-truth disparity", GT_disp_vis); } + //! [visualization] Mat raw_disp_vis; getDisparityVis(left_disp,raw_disp_vis,vis_mult); namedWindow("raw disparity", WINDOW_AUTOSIZE); @@ -301,6 +324,7 @@ int main(int argc, char** argv) namedWindow("filtered disparity", WINDOW_AUTOSIZE); imshow("filtered disparity", filtered_disp_vis); waitKey(); + //! [visualization] } return 0; diff --git a/modules/ximgproc/src/disparity_filters.cpp b/modules/ximgproc/src/disparity_filters.cpp index ec4460ae8..63db08316 100644 --- a/modules/ximgproc/src/disparity_filters.cpp +++ b/modules/ximgproc/src/disparity_filters.cpp @@ -49,17 +49,22 @@ using std::vector; class DisparityWLSFilterImpl : public DisparityWLSFilter { protected: - double lambda,sigma_color; + int left_offset, right_offset, top_offset, bottom_offset; + Rect valid_disp_ROI; + Rect right_view_valid_disp_ROI; + int min_disp; bool use_confidence; Mat confidence_map; + + double lambda,sigma_color; int LRC_thresh,depth_discontinuity_radius; float depth_discontinuity_roll_off_factor; float resize_factor; int num_stripes; - void init(double _lambda, double _sigma_color, bool _use_confidence); - void computeDepthDiscontinuityMaps(Mat& left_disp, Mat& right_disp, Mat& left_dst, Mat& right_dst, Rect ROI); - void computeConfidenceMap(InputArray left_disp, InputArray right_disp, Rect ROI); + void init(double _lambda, double _sigma_color, bool _use_confidence, int l_offs, int r_offs, int t_offs, int b_offs, int _min_disp); + void computeDepthDiscontinuityMaps(Mat& left_disp, Mat& right_disp, Mat& left_dst, Mat& right_dst); + void computeConfidenceMap(InputArray left_disp, InputArray right_disp); protected: struct ComputeDiscontinuityAwareLRC_ParBody : public ParallelLoopBody @@ -99,13 +104,13 @@ protected: void boxFilterOp(Mat& src,Mat& dst) { - int rad = (int)ceil(resize_factor*depth_discontinuity_radius); + int rad = depth_discontinuity_radius; boxFilter(src,dst,CV_32F,Size(2*rad+1,2*rad+1),Point(-1,-1)); } void sqrBoxFilterOp(Mat& src,Mat& dst) { - int rad = (int)ceil(resize_factor*depth_discontinuity_radius); + int rad = depth_discontinuity_radius; sqrBoxFilter(src,dst,CV_32F,Size(2*rad+1,2*rad+1),Point(-1,-1)); } @@ -115,22 +120,33 @@ protected: } public: - static Ptr create(bool _use_confidence); - void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right, InputArray); + static Ptr create(bool _use_confidence, int l_offs, int r_offs, int t_offs, int b_offs, int min_disp); + void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right, Rect ROI, InputArray); double getLambda() {return lambda;} void setLambda(double _lambda) {lambda = _lambda;} + double getSigmaColor() {return sigma_color;} void setSigmaColor(double _sigma_color) {sigma_color = _sigma_color;} - Mat getConfidenceMap() {return confidence_map;} + int getLRCthresh() {return LRC_thresh;} void setLRCthresh(int _LRC_thresh) {LRC_thresh = _LRC_thresh;} + int getDepthDiscontinuityRadius() {return depth_discontinuity_radius;} void setDepthDiscontinuityRadius(int _disc_radius) {depth_discontinuity_radius = _disc_radius;} + + Mat getConfidenceMap() {return confidence_map;} + Rect getROI() {return valid_disp_ROI;} }; -void DisparityWLSFilterImpl::init(double _lambda, double _sigma_color, bool _use_confidence) +void DisparityWLSFilterImpl::init(double _lambda, double _sigma_color, bool _use_confidence, int l_offs, int r_offs, int t_offs, int b_offs, int _min_disp) { + left_offset = l_offs; right_offset = r_offs; + top_offset = t_offs; bottom_offset = b_offs; + min_disp = _min_disp; + valid_disp_ROI = Rect(); + right_view_valid_disp_ROI = Rect(); + min_disp=0; lambda = _lambda; sigma_color = _sigma_color; use_confidence = _use_confidence; @@ -142,11 +158,10 @@ void DisparityWLSFilterImpl::init(double _lambda, double _sigma_color, bool _use num_stripes = getNumThreads(); } -void DisparityWLSFilterImpl::computeDepthDiscontinuityMaps(Mat& left_disp, Mat& right_disp, Mat& left_dst, Mat& right_dst, Rect ROI) +void DisparityWLSFilterImpl::computeDepthDiscontinuityMaps(Mat& left_disp, Mat& right_disp, Mat& left_dst, Mat& right_dst) { - Rect right_ROI(left_disp.cols-(ROI.x+ROI.width),ROI.y,ROI.width,ROI.height); - Mat left_disp_ROI (left_disp, ROI); - Mat right_disp_ROI(right_disp,right_ROI); + Mat left_disp_ROI (left_disp, valid_disp_ROI); + Mat right_disp_ROI(right_disp,right_view_valid_disp_ROI); Mat ldisp,rdisp,ldisp_squared,rdisp_squared; { @@ -171,36 +186,37 @@ void DisparityWLSFilterImpl::computeDepthDiscontinuityMaps(Mat& left_disp, Mat& left_dst = Mat::zeros(left_disp.rows,left_disp.cols,CV_32F); right_dst = Mat::zeros(right_disp.rows,right_disp.cols,CV_32F); - Mat left_dst_ROI (left_dst,ROI); - Mat right_dst_ROI(right_dst,right_ROI); + Mat left_dst_ROI (left_dst,valid_disp_ROI); + Mat right_dst_ROI(right_dst,right_view_valid_disp_ROI); parallel_for_(Range(0,num_stripes),ComputeDepthDisc_ParBody(*this,ldisp,ldisp_squared,left_dst_ROI ,num_stripes)); parallel_for_(Range(0,num_stripes),ComputeDepthDisc_ParBody(*this,rdisp,rdisp_squared,right_dst_ROI,num_stripes)); } -void DisparityWLSFilterImpl::computeConfidenceMap(InputArray left_disp, InputArray right_disp, Rect ROI) +void DisparityWLSFilterImpl::computeConfidenceMap(InputArray left_disp, InputArray right_disp) { Mat ldisp = left_disp.getMat(); Mat rdisp = right_disp.getMat(); Mat depth_discontinuity_map_left,depth_discontinuity_map_right; - computeDepthDiscontinuityMaps(ldisp,rdisp,depth_discontinuity_map_left,depth_discontinuity_map_right,ROI); + right_view_valid_disp_ROI = Rect(ldisp.cols-(valid_disp_ROI.x+valid_disp_ROI.width),valid_disp_ROI.y, + valid_disp_ROI.width,valid_disp_ROI.height); + computeDepthDiscontinuityMaps(ldisp,rdisp,depth_discontinuity_map_left,depth_discontinuity_map_right); - Rect right_ROI(ldisp.cols-(ROI.x+ROI.width),ROI.y,ROI.width,ROI.height); confidence_map = depth_discontinuity_map_left; - parallel_for_(Range(0,num_stripes),ComputeDiscontinuityAwareLRC_ParBody(*this,ldisp,rdisp, depth_discontinuity_map_left,depth_discontinuity_map_right,confidence_map,ROI,right_ROI,num_stripes)); + parallel_for_(Range(0,num_stripes),ComputeDiscontinuityAwareLRC_ParBody(*this,ldisp,rdisp, depth_discontinuity_map_left,depth_discontinuity_map_right,confidence_map,valid_disp_ROI,right_view_valid_disp_ROI,num_stripes)); confidence_map = 255.0f*confidence_map; } -Ptr DisparityWLSFilterImpl::create(bool _use_confidence) +Ptr DisparityWLSFilterImpl::create(bool _use_confidence, int l_offs=0, int r_offs=0, int t_offs=0, int b_offs=0, int min_disp=0) { DisparityWLSFilterImpl *wls = new DisparityWLSFilterImpl(); - wls->init(8000.0,1.0,_use_confidence); + wls->init(8000.0,1.0,_use_confidence,l_offs, r_offs, t_offs, b_offs, min_disp); return Ptr(wls); } -void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right, InputArray) +void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right, Rect ROI, InputArray) { CV_Assert( !disparity_map_left.empty() && (disparity_map_left.depth() == CV_16S) && (disparity_map_left.channels() == 1) ); CV_Assert( !left_view.empty() && (left_view.depth() == CV_8U) && (left_view.channels() == 3 || left_view.channels() == 1) ); @@ -209,6 +225,12 @@ void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray le resize_factor = disparity_map_left.cols()/(float)left_view.cols(); else resize_factor = 1.0; + if(ROI.area()!=0) /* user provided a ROI */ + valid_disp_ROI = ROI; + else + valid_disp_ROI = Rect(left_offset,top_offset, + disparity_map_left.cols()-left_offset-right_offset, + disparity_map_left.rows()-top_offset-bottom_offset); if(!use_confidence) { @@ -220,13 +242,16 @@ void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray le float y_ratio = src_full_size.rows/(float)disp_full_size.rows; resize(disp_full_size,disp_full_size,src_full_size.size()); disp_full_size = disp_full_size*x_ratio; - ROI = Rect((int)(ROI.x*x_ratio),(int)(ROI.y*y_ratio),(int)(ROI.width*x_ratio),(int)(ROI.height*y_ratio)); + ROI = Rect((int)(valid_disp_ROI.x*x_ratio), (int)(valid_disp_ROI.y*y_ratio), + (int)(valid_disp_ROI.width*x_ratio),(int)(valid_disp_ROI.height*y_ratio)); } + else + ROI = valid_disp_ROI; disp = Mat(disp_full_size,ROI); src = Mat(src_full_size ,ROI); filtered_disparity_map.create(disp_full_size.size(), disp_full_size.type()); Mat& dst_full_size = filtered_disparity_map.getMatRef(); - dst_full_size = Scalar(-16); + dst_full_size = Scalar(16*(min_disp-1)); dst = Mat(dst_full_size,ROI); Mat filtered_disp; fastGlobalSmootherFilter(src,disp,filtered_disp,lambda,sigma_color); @@ -237,7 +262,7 @@ void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray le CV_Assert( !disparity_map_right.empty() && (disparity_map_right.depth() == CV_16S) && (disparity_map_right.channels() == 1) ); CV_Assert( (disparity_map_left.cols() == disparity_map_right.cols()) ); CV_Assert( (disparity_map_left.rows() == disparity_map_right.rows()) ); - computeConfidenceMap(disparity_map_left,disparity_map_right,ROI); + computeConfidenceMap(disparity_map_left,disparity_map_right); Mat disp_full_size = disparity_map_left.getMat(); Mat src_full_size = left_view.getMat(); if(disp_full_size.size!=src_full_size.size) @@ -247,13 +272,16 @@ void DisparityWLSFilterImpl::filter(InputArray disparity_map_left, InputArray le resize(disp_full_size,disp_full_size,src_full_size.size()); disp_full_size = disp_full_size*x_ratio; resize(confidence_map,confidence_map,src_full_size.size()); - ROI = Rect((int)(ROI.x*x_ratio),(int)(ROI.y*y_ratio),(int)(ROI.width*x_ratio),(int)(ROI.height*y_ratio)); + ROI = Rect((int)(valid_disp_ROI.x*x_ratio), (int)(valid_disp_ROI.y*y_ratio), + (int)(valid_disp_ROI.width*x_ratio),(int)(valid_disp_ROI.height*y_ratio)); } + else + ROI = valid_disp_ROI; disp = Mat(disp_full_size,ROI); src = Mat(src_full_size ,ROI); filtered_disparity_map.create(disp_full_size.size(), disp_full_size.type()); Mat& dst_full_size = filtered_disparity_map.getMatRef(); - dst_full_size = Scalar(-16); + dst_full_size = Scalar(16*(min_disp-1)); dst = Mat(dst_full_size,ROI); Mat conf(confidence_map,ROI); @@ -355,7 +383,73 @@ void DisparityWLSFilterImpl::ParallelMatOp_ParBody::operator() (const Range& ran } CV_EXPORTS_W -Ptr createDisparityWLSFilter(bool use_confidence) +Ptr createDisparityWLSFilter(Ptr matcher_left) +{ + Ptr wls; + matcher_left->setDisp12MaxDiff(1000000); + matcher_left->setSpeckleWindowSize(0); + + int min_disp = matcher_left->getMinDisparity(); + int num_disp = matcher_left->getNumDisparities(); + int wsize = matcher_left->getBlockSize(); + int wsize2 = wsize/2; + + if(Ptr bm = matcher_left.dynamicCast()) + { + bm->setTextureThreshold(0); + bm->setUniquenessRatio(0); + wls = DisparityWLSFilterImpl::create(true,max(0,min_disp+num_disp)+wsize2,max(0,-min_disp)+wsize2,wsize2,wsize2,min_disp); + wls->setDepthDiscontinuityRadius((int)ceil(0.33*wsize)); + } + else if(Ptr sgbm = matcher_left.dynamicCast()) + { + sgbm->setUniquenessRatio(0); + wls = DisparityWLSFilterImpl::create(true,max(0,min_disp+num_disp),max(0,-min_disp),0,0,min_disp); + wls->setDepthDiscontinuityRadius((int)ceil(0.5*wsize)); + } + else + CV_Error(Error::StsBadArg, "DisparityWLSFilter natively supports only StereoBM and StereoSGBM"); + + return wls; +} + +CV_EXPORTS_W +Ptr createRightMatcher(Ptr matcher_left) +{ + int min_disp = matcher_left->getMinDisparity(); + int num_disp = matcher_left->getNumDisparities(); + int wsize = matcher_left->getBlockSize(); + if(Ptr bm = matcher_left.dynamicCast()) + { + Ptr right_bm = StereoBM::create(num_disp,wsize); + right_bm->setMinDisparity(-(min_disp+num_disp)+1); + right_bm->setTextureThreshold(0); + right_bm->setUniquenessRatio(0); + right_bm->setDisp12MaxDiff(1000000); + right_bm->setSpeckleWindowSize(0); + return right_bm; + } + else if(Ptr sgbm = matcher_left.dynamicCast()) + { + Ptr right_sgbm = StereoSGBM::create(-(min_disp+num_disp)+1,num_disp,wsize); + right_sgbm->setUniquenessRatio(0); + right_sgbm->setP1(sgbm->getP1()); + right_sgbm->setP2(sgbm->getP2()); + right_sgbm->setMode(sgbm->getMode()); + right_sgbm->setPreFilterCap(sgbm->getPreFilterCap()); + right_sgbm->setDisp12MaxDiff(1000000); + right_sgbm->setSpeckleWindowSize(0); + return right_sgbm; + } + else + { + CV_Error(Error::StsBadArg, "createRightMatcher supports only StereoBM and StereoSGBM"); + return Ptr(); + } +} + +CV_EXPORTS_W +Ptr createDisparityWLSFilterGeneric(bool use_confidence) { return Ptr(DisparityWLSFilterImpl::create(use_confidence)); } diff --git a/modules/ximgproc/test/test_disparity_wls_filter.cpp b/modules/ximgproc/test/test_disparity_wls_filter.cpp index 0170a9002..dc2d44bf4 100644 --- a/modules/ximgproc/test/test_disparity_wls_filter.cpp +++ b/modules/ximgproc/test/test_disparity_wls_filter.cpp @@ -83,10 +83,10 @@ TEST(DisparityWLSFilterTest, ReferenceAccuracy) cv::setNumThreads(cv::getNumberOfCPUs()); Mat res; - Ptr wls_filter = createDisparityWLSFilter(true); + Ptr wls_filter = createDisparityWLSFilterGeneric(true); wls_filter->setLambda(8000.0); wls_filter->setSigmaColor(0.5); - wls_filter->filter(left_disp,left,res,ROI,right_disp); + wls_filter->filter(left_disp,left,res,right_disp,ROI); double MSE = computeMSE(GT,res,ROI); double BadPercent = computeBadPixelPercent(GT,res,ROI); @@ -134,17 +134,17 @@ TEST_P(DisparityWLSFilterTest, MultiThreadReproducibility) double lambda = rng.uniform(100.0, 10000.0); double sigma = rng.uniform(1.0, 100.0); - Ptr wls_filter = createDisparityWLSFilter(use_conf); + Ptr wls_filter = createDisparityWLSFilterGeneric(use_conf); wls_filter->setLambda(lambda); wls_filter->setSigmaColor(sigma); cv::setNumThreads(cv::getNumberOfCPUs()); Mat resMultiThread; - wls_filter->filter(left_disp,left,resMultiThread,ROI,right_disp); + wls_filter->filter(left_disp,left,resMultiThread,right_disp,ROI); cv::setNumThreads(1); Mat resSingleThread; - wls_filter->filter(left_disp,left,resSingleThread,ROI,right_disp); + wls_filter->filter(left_disp,left,resSingleThread,right_disp,ROI); EXPECT_LE(cv::norm(resSingleThread, resMultiThread, NORM_INF), MAX_DIF); EXPECT_LE(cv::norm(resSingleThread, resMultiThread, NORM_L1), MAX_MEAN_DIF*left.total()); diff --git a/modules/ximgproc/tutorials/disparity_filtering.markdown b/modules/ximgproc/tutorials/disparity_filtering.markdown new file mode 100644 index 000000000..4ca559f83 --- /dev/null +++ b/modules/ximgproc/tutorials/disparity_filtering.markdown @@ -0,0 +1,76 @@ +Disparity map post-filtering {#tutorial_ximgproc_disparity_filtering} +============================ + +Introduction +------------ + +Stereo matching algorithms, especially highly-optimized ones that are intended for real-time processing +on CPU, tend to make quite a few errors on challenging sequences. These errors are usually concentrated +in uniform texture-less areas, half-occlusions and regions near depth discontinuities. One way of dealing +with stereo-matching errors is to use various techniques of detecting potentially inaccurate disparity +values and invalidate them, therefore making the disparity map semi-sparse. Several such techniques are +already implemented in the StereoBM and StereoSGBM algorithms. Another way would be to use some kind of +filtering procedure to align the disparity map edges with those of the source image and to propagate +the disparity values from high- to low-confidence regions like half-occlusions. Recent advances in +edge-aware filtering have enabled performing such post-filtering under the constraints of real-time +processing on CPU. + +In this tutorial you will learn how to use the disparity map post-filtering to improve the results +of StereoBM and StereoSGBM algorithms. + +Source Stereoscopic Image +------------------------- + +![Left view](images/ambush_5_left.jpg) +![Right view](images/ambush_5_right.jpg) + +Source Code +----------- + +We will be using snippets from the example application, that can be downloaded [here ](https://github.com/Itseez/opencv_contrib/blob/master/modules/ximgproc/samples/disparity_filtering.cpp). + +Explanation +----------- + +The provided example has several options that yield different trade-offs between the speed and +the quality of the resulting disparity map. Both the speed and the quality are measured if the user +has provided the ground-truth disparity map. In this tutorial we will take a detailed look at the +default pipeline, that was designed to provide the best possible quality under the constraints of +real-time processing on CPU. + +-# **Load left and right views** + @snippet ximgproc/samples/disparity_filtering.cpp load_views + We start by loading the source stereopair. For this tutorial we will take a somewhat challenging + example from the MPI-Sintel dataset with a lot of texture-less regions. + +-# **Prepare the views for matching** + @snippet ximgproc/samples/disparity_filtering.cpp downscale + We perform downscaling of the views to speed-up the matching stage at the cost of minor + quality degradation. To get the best possible quality downscaling should be avoided. + +-# **Perform matching and create the filter instance** + @snippet ximgproc/samples/disparity_filtering.cpp matching + We are using StereoBM for faster processing. If speed is not critical, though, + StereoSGBM would provide better quality. The filter instance is created by providing + the StereoMatcher instance that we intend to use. Another matcher instance is + returned by the createRightMatcher function. These two matcher instances are then + used to compute disparity maps both for the left and right views, that are required + by the filter. + +-# **Perform filtering** + @snippet ximgproc/samples/disparity_filtering.cpp filtering + Disparity maps computed by the respective matcher instances, as well as the source left view + are passed to the filter. Note that we are using the original non-downscaled view to guide the + filtering process. The disparity map is automatically upscaled in an edge-aware fashion to match + the original view resolution. The result is stored in filtered_disp. + +-# **Visualize the disparity maps** + @snippet ximgproc/samples/disparity_filtering.cpp visualization + We use a convenience function getDisparityVis to visualize the disparity maps. The second parameter + defines the contrast (all disparity values are scaled by this value in the visualization). + +Results +------- + +![Result of the StereoBM](images/ambush_5_bm.png) +![Result of the demonstrated pipeline (StereoBM on downscaled views with post-filtering)](images/ambush_5_bm_with_filter.png) diff --git a/modules/ximgproc/tutorials/images/ambush_5_bm.png b/modules/ximgproc/tutorials/images/ambush_5_bm.png new file mode 100644 index 000000000..0f3391813 Binary files /dev/null and b/modules/ximgproc/tutorials/images/ambush_5_bm.png differ diff --git a/modules/ximgproc/tutorials/images/ambush_5_bm_with_filter.png b/modules/ximgproc/tutorials/images/ambush_5_bm_with_filter.png new file mode 100644 index 000000000..3a585238d Binary files /dev/null and b/modules/ximgproc/tutorials/images/ambush_5_bm_with_filter.png differ diff --git a/modules/ximgproc/tutorials/images/ambush_5_left.jpg b/modules/ximgproc/tutorials/images/ambush_5_left.jpg new file mode 100644 index 000000000..0fb11f5e3 Binary files /dev/null and b/modules/ximgproc/tutorials/images/ambush_5_left.jpg differ diff --git a/modules/ximgproc/tutorials/images/ambush_5_right.jpg b/modules/ximgproc/tutorials/images/ambush_5_right.jpg new file mode 100644 index 000000000..2f9f4f1ef Binary files /dev/null and b/modules/ximgproc/tutorials/images/ambush_5_right.jpg differ