From 21d89cc4206a32c6de274f80ea7f709d82ef1187 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 18 May 2011 07:11:45 +0000 Subject: [PATCH] added leaveBiggestComponent function into opencv_stitching --- modules/stitching/main.cpp | 17 +++-- modules/stitching/matchers.cpp | 25 ++++--- modules/stitching/matchers.hpp | 9 +-- modules/stitching/motion_estimators.cpp | 94 ++++++++++++++++++++----- modules/stitching/motion_estimators.hpp | 9 ++- 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/modules/stitching/main.cpp b/modules/stitching/main.cpp index 15c910daf2..4916503449 100644 --- a/modules/stitching/main.cpp +++ b/modules/stitching/main.cpp @@ -15,7 +15,7 @@ void printUsage() cout << "Usage: opencv_stitching img1 img2 [...imgN]\n" << "\t[--matchconf ]\n" << "\t[--ba (ray|focal_ray)]\n" - << "\t[--ba_thresh ]\n" + << "\t[--conf_thresh ]\n" << "\t[--wavecorrect (no|yes)]\n" << "\t[--warp (plane|cylindrical|spherical)]\n" << "\t[--seam (no|voronoi|graphcut)]\n" @@ -23,7 +23,7 @@ void printUsage() << "\t[--output ]\n\n"; cout << "--matchconf\n" << "\tGood values are in [0.2, 0.8] range usually.\n\n"; - cout << "--ba_thresh\n" + cout << "--conf_thresh\n" << "\tGood values are in [0.3, 1.0] range usually.\n"; } @@ -34,7 +34,7 @@ int main(int argc, char* argv[]) vector images; string result_name = "result.png"; int ba_space = BundleAdjuster::FOCAL_RAY_SPACE; - float ba_thresh = 1.f; + float conf_thresh = 1.f; bool wave_correct = true; int warp_type = Warper::SPHERICAL; bool user_match_conf = false; @@ -74,9 +74,9 @@ int main(int argc, char* argv[]) } i++; } - else if (string(argv[i]) == "--ba_thresh") + else if (string(argv[i]) == "--conf_thresh") { - ba_thresh = static_cast(atof(argv[i + 1])); + conf_thresh = static_cast(atof(argv[i + 1])); i++; } else if (string(argv[i]) == "--wavecorrect") @@ -154,7 +154,7 @@ int main(int argc, char* argv[]) } } - const int num_images = static_cast(images.size()); + int num_images = static_cast(images.size()); if (num_images < 2) { cout << "Need more images\n"; @@ -173,6 +173,9 @@ int main(int argc, char* argv[]) matcher = BestOf2NearestMatcher(true, match_conf); matcher(images, features, pairwise_matches); + leaveBiggestComponent(images, features, pairwise_matches, conf_thresh); + num_images = static_cast(images.size()); + LOGLN("Estimating rotations..."); HomographyBasedEstimator estimator; vector cameras; @@ -187,7 +190,7 @@ int main(int argc, char* argv[]) } LOGLN("Bundle adjustment..."); - BundleAdjuster adjuster(ba_space, ba_thresh); + BundleAdjuster adjuster(ba_space, conf_thresh); adjuster(images, features, pairwise_matches, cameras); if (wave_correct) diff --git a/modules/stitching/matchers.cpp b/modules/stitching/matchers.cpp index 9dd44b6264..b17ea45404 100644 --- a/modules/stitching/matchers.cpp +++ b/modules/stitching/matchers.cpp @@ -123,22 +123,19 @@ void SurfFeaturesFinder::find(const vector &images, vector & ////////////////////////////////////////////////////////////////////////////// -MatchesInfo::MatchesInfo() : src_img_idx(-1), dst_img_idx(-1), num_inliers(0) {} - - -MatchesInfo::MatchesInfo(const MatchesInfo &other) -{ - *this = other; -} +MatchesInfo::MatchesInfo() : src_img_idx(-1), dst_img_idx(-1), num_inliers(0), confidence(0) {} +MatchesInfo::MatchesInfo(const MatchesInfo &other) { *this = other; } const MatchesInfo& MatchesInfo::operator =(const MatchesInfo &other) { src_img_idx = other.src_img_idx; dst_img_idx = other.dst_img_idx; matches = other.matches; + inliers_mask = other.inliers_mask; num_inliers = other.num_inliers; H = other.H.clone(); + confidence = other.confidence; return *this; } @@ -303,14 +300,14 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features Mat dst_points(1, matches_info.matches.size(), CV_32FC2); for (size_t i = 0; i < matches_info.matches.size(); ++i) { - const DMatch& r = matches_info.matches[i]; + const DMatch& m = matches_info.matches[i]; - Point2f p = features1.keypoints[r.queryIdx].pt; + Point2f p = features1.keypoints[m.queryIdx].pt; p.x -= img1.cols * 0.5f; p.y -= img1.rows * 0.5f; src_points.at(0, i) = p; - p = features2.keypoints[r.trainIdx].pt; + p = features2.keypoints[m.trainIdx].pt; p.x -= img2.cols * 0.5f; p.y -= img2.rows * 0.5f; dst_points.at(0, i) = p; @@ -325,6 +322,8 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features if (matches_info.inliers_mask[i]) matches_info.num_inliers++; + matches_info.confidence = matches_info.num_inliers / (8 + 0.3*matches_info.matches.size()); + // Check if we should try to refine motion if (matches_info.num_inliers < num_matches_thresh2_) return; @@ -338,14 +337,14 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features if (!matches_info.inliers_mask[i]) continue; - const DMatch& r = matches_info.matches[i]; + const DMatch& m = matches_info.matches[i]; - Point2f p = features1.keypoints[r.queryIdx].pt; + Point2f p = features1.keypoints[m.queryIdx].pt; p.x -= img1.cols * 0.5f; p.y -= img2.rows * 0.5f; src_points.at(0, inlier_idx) = p; - p = features2.keypoints[r.trainIdx].pt; + p = features2.keypoints[m.trainIdx].pt; p.x -= img2.cols * 0.5f; p.y -= img2.rows * 0.5f; dst_points.at(0, inlier_idx) = p; diff --git a/modules/stitching/matchers.hpp b/modules/stitching/matchers.hpp index 9b6e4db523..c3788d0e21 100644 --- a/modules/stitching/matchers.hpp +++ b/modules/stitching/matchers.hpp @@ -43,11 +43,12 @@ struct MatchesInfo MatchesInfo(const MatchesInfo &other); const MatchesInfo& operator =(const MatchesInfo &other); - int src_img_idx, dst_img_idx; // Optional images indices + int src_img_idx, dst_img_idx; // Images indices (optional) std::vector matches; - std::vector inliers_mask; - int num_inliers; // Number of geometrically consistent matches - cv::Mat H; // Homography + std::vector inliers_mask; // Geometrically consistent matches mask + int num_inliers; // Number of geometrically consistent matches + cv::Mat H; // Estimated homography + double confidence; // Confidence two images are from the same panorama }; diff --git a/modules/stitching/motion_estimators.cpp b/modules/stitching/motion_estimators.cpp index 08be4578b6..4f95e52226 100644 --- a/modules/stitching/motion_estimators.cpp +++ b/modules/stitching/motion_estimators.cpp @@ -13,12 +13,7 @@ using namespace cv; CameraParams::CameraParams() : focal(1), R(Mat::eye(3, 3, CV_64F)), t(Mat::zeros(3, 1, CV_64F)) {} - -CameraParams::CameraParams(const CameraParams &other) -{ - *this = other; -} - +CameraParams::CameraParams(const CameraParams &other) { *this = other; } const CameraParams& CameraParams::operator =(const CameraParams &other) { @@ -84,13 +79,14 @@ void HomographyBasedEstimator::estimate(const vector &images, const vector< int pair_idx = i * num_images + j; if (pairwise_matches[pair_idx].H.empty()) continue; + double f_to, f_from; bool f_to_ok, f_from_ok; focalsFromHomography(pairwise_matches[pair_idx].H.inv(), f_to, f_from, f_to_ok, f_from_ok); - if (f_from_ok) - focals.push_back(f_from); - if (f_to_ok) - focals.push_back(f_to); + + if (f_from_ok) focals.push_back(f_from); + if (f_to_ok) focals.push_back(f_to); + if (f_from_ok && f_to_ok) { is_focal_estimated[i] = true; @@ -98,6 +94,7 @@ void HomographyBasedEstimator::estimate(const vector &images, const vector< } } } + is_focals_estimated_ = true; for (int i = 0; i < num_images; ++i) is_focals_estimated_ = is_focals_estimated_ && is_focal_estimated[i]; @@ -132,9 +129,12 @@ void BundleAdjuster::estimate(const vector &images, const vector(i * 4, 0) = cameras[i].focal; + svd(cameras[i].R, SVD::FULL_UV); Mat R = svd.u * svd.vt; - if (determinant(R) < 0) R *= -1; + if (determinant(R) < 0) + R *= -1; + Mat rvec; Rodrigues(R, rvec); CV_Assert(rvec.type() == CV_32F); cameras_.at(i * 4 + 1, 0) = rvec.at(0, 0); @@ -142,20 +142,19 @@ void BundleAdjuster::estimate(const vector &images, const vector(i * 4 + 3, 0) = rvec.at(2, 0); } + // Select only consistent image pairs for futher adjustment edges_.clear(); for (int i = 0; i < num_images_ - 1; ++i) { for (int j = i + 1; j < num_images_; ++j) - { - int pair_idx = i * num_images_ + j; - const MatchesInfo& mi = pairwise_matches_[pair_idx]; - float ni = static_cast(mi.num_inliers); - float nf = static_cast(mi.matches.size()); - if (ni / (8.f + 0.3f * nf) > dist_thresh_) + { + const MatchesInfo& matches_info = pairwise_matches_[i * num_images_ + j]; + if (matches_info.confidence > conf_thresh_) edges_.push_back(make_pair(i, j)); } } + // Compute number of correspondences total_num_matches_ = 0; for (size_t i = 0; i < edges_.size(); ++i) total_num_matches_ += static_cast(pairwise_matches[edges_[i].first * num_images_ + edges_[i].second].num_inliers); @@ -369,7 +368,6 @@ void waveCorrect(vector &rmats) normalize(r0, r0); r1.cross(r0).copyTo(r2); - if (determinant(R) < 0) R *= -1; @@ -380,6 +378,64 @@ void waveCorrect(vector &rmats) ////////////////////////////////////////////////////////////////////////////// +void leaveBiggestComponent(vector &images, vector &features, + vector &pairwise_matches, float conf_threshold) +{ + const int num_images = static_cast(images.size()); + + DjSets comps(num_images); + for (int i = 0; i < num_images; ++i) + { + for (int j = 0; j < num_images; ++j) + { + if (pairwise_matches[i*num_images + j].confidence < conf_threshold) + continue; + int comp1 = comps.find(i); + int comp2 = comps.find(j); + if (comp1 != comp2) + comps.merge(comp1, comp2); + } + } + + int max_comp = max_element(comps.size.begin(), comps.size.end()) - comps.size.begin(); + + vector indices; + vector indices_removed; + for (int i = 0; i < num_images; ++i) + if (comps.find(i) == max_comp) + indices.push_back(i); + else + indices_removed.push_back(i); + + vector images_subset; + vector features_subset; + vector pairwise_matches_subset; + for (size_t i = 0; i < indices.size(); ++i) + { + images_subset.push_back(images[indices[i]]); + features_subset.push_back(features[indices[i]]); + for (size_t j = 0; j < indices.size(); ++j) + { + pairwise_matches_subset.push_back(pairwise_matches[indices[i]*num_images + indices[j]]); + pairwise_matches_subset.back().src_img_idx = i; + pairwise_matches_subset.back().dst_img_idx = j; + } + } + + if (static_cast(images_subset.size()) == num_images) + return; + + LOG("Removed some images, because can't match them: ("); + LOG(indices_removed[0]); + for (size_t i = 1; i < indices_removed.size(); ++i) LOG(", " << indices_removed[i]); + LOGLN(")"); + + images = images_subset; + features = features_subset; + pairwise_matches = pairwise_matches_subset; +} + + void findMaxSpanningTree(int num_images, const vector &pairwise_matches, Graph &span_tree, vector ¢ers) { @@ -391,6 +447,8 @@ void findMaxSpanningTree(int num_images, const vector &pairwise_mat { for (int j = 0; j < num_images; ++j) { + if (pairwise_matches[i * num_images + j].H.empty()) + continue; float conf = static_cast(pairwise_matches[i * num_images + j].num_inliers); graph.addEdge(i, j, conf); edges.push_back(GraphEdge(i, j, conf)); diff --git a/modules/stitching/motion_estimators.hpp b/modules/stitching/motion_estimators.hpp index 97a0643872..3a782a297f 100644 --- a/modules/stitching/motion_estimators.hpp +++ b/modules/stitching/motion_estimators.hpp @@ -52,8 +52,8 @@ class BundleAdjuster : public Estimator public: enum { RAY_SPACE, FOCAL_RAY_SPACE }; - BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float dist_thresh = 1.f) - : cost_space_(cost_space), dist_thresh_(dist_thresh) {} + BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float conf_thresh = 1.f) + : cost_space_(cost_space), conf_thresh_(conf_thresh) {} private: void estimate(const std::vector &images, const std::vector &features, @@ -71,7 +71,7 @@ private: std::vector > edges_; int cost_space_; - float dist_thresh_; + float conf_thresh_; cv::Mat err_, err1_, err2_; cv::Mat J_; }; @@ -83,6 +83,9 @@ void waveCorrect(std::vector &rmats); ////////////////////////////////////////////////////////////////////////////// // Auxiliary functions +void leaveBiggestComponent(std::vector &images, std::vector &features, + std::vector &pairwise_matches, float conf_threshold); + void findMaxSpanningTree(int num_images, const std::vector &pairwise_matches, Graph &span_tree, std::vector ¢ers);