Refactored stitching module

modules/stitching/include/opencv2/stitching/stitcher.hpp

@@ -51,6 +51,7 @@
 #include "detail/exposure_compensate.hpp"
 #include "detail/seam_finders.hpp"
 #include "detail/blenders.hpp"
+#include "detail/camera.hpp"
 
 namespace cv {
 
@@ -78,8 +79,11 @@ public:
     double panoConfidenceThresh() const { return conf_thresh_; }
     void setPanoConfidenceThresh(double conf_thresh) { conf_thresh_ = conf_thresh; }
 
-    bool horizontalStrightening() const { return horiz_stright_; }
+    bool waveCorrection() const { return do_wave_correct_; }
-    void setHorizontalStrightening(bool flag) { horiz_stright_ = flag; }
+    void setWaveCorrection(bool flag) { do_wave_correct_ = flag; }
+
+    detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; }
+    void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; }
 
     Ptr<detail::FeaturesFinder> featuresFinder() { return features_finder_; }
     const Ptr<detail::FeaturesFinder> featuresFinder() const { return features_finder_; }
@@ -96,9 +100,6 @@ public:
     void setBundleAdjuster(Ptr<detail::BundleAdjusterBase> bundle_adjuster)
         { bundle_adjuster_ = bundle_adjuster; }
 
-    detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; }
-    void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; }
-
     Ptr<WarperCreator> warper() { return warper_; }
     const Ptr<WarperCreator> warper() const { return warper_; }
     void setWarper(Ptr<WarperCreator> warper) { warper_ = warper; }
@@ -119,19 +120,35 @@ public:
 private:
     Stitcher() {}
 
+    Status matchImages();
+    void estimateCameraParams();
+    Status composePanorama(cv::Mat &pano);
+
     double registr_resol_;
     double seam_est_resol_;
     double compose_resol_;
     double conf_thresh_;
-    bool horiz_stright_;
     Ptr<detail::FeaturesFinder> features_finder_;
     Ptr<detail::FeaturesMatcher> features_matcher_;
     Ptr<detail::BundleAdjusterBase> bundle_adjuster_;
+    bool do_wave_correct_;
     detail::WaveCorrectKind wave_correct_kind_;
     Ptr<WarperCreator> warper_;
     Ptr<detail::ExposureCompensator> exposure_comp_;
     Ptr<detail::SeamFinder> seam_finder_;
     Ptr<detail::Blender> blender_;
+
+    std::vector<cv::Mat> imgs_;
+    std::vector<cv::Size> full_img_sizes_;
+    std::vector<detail::ImageFeatures> features_;
+    std::vector<detail::MatchesInfo> pairwise_matches_;
+    std::vector<cv::Mat> seam_est_imgs_;
+    std::vector<int> indices_;
+    std::vector<detail::CameraParams> cameras_;
+    double work_scale_;
+    double seam_scale_;
+    double seam_work_aspect_;
+    double warped_image_scale_;
 };
 
 } // namespace cv

modules/stitching/src/stitcher.cpp

@@ -53,10 +53,10 @@ Stitcher Stitcher::createDefault(bool try_use_gpu)
     stitcher.setSeamEstimationResol(0.1);
     stitcher.setCompositingResol(ORIG_RESOL);
     stitcher.setPanoConfidenceThresh(1);
-    stitcher.setHorizontalStrightening(true);
+    stitcher.setWaveCorrection(true);
+    stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ);
     stitcher.setFeaturesMatcher(new detail::BestOf2NearestMatcher(try_use_gpu));
     stitcher.setBundleAdjuster(new detail::BundleAdjusterRay());
-    stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ);
 
 #ifndef ANDROID
     if (try_use_gpu && gpu::getCudaEnabledDeviceCount() > 0)
@@ -80,70 +80,83 @@ Stitcher Stitcher::createDefault(bool try_use_gpu)
 }
 
 
-Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
+Stitcher::Status Stitcher::stitch(InputArray imgs, OutputArray pano)
 {
-    // TODO split this func
+    int64 app_start_time = getTickCount();
 
-    vector<Mat> imgs;
+    imgs.getMatVector(imgs_);
-    imgs_.getMatVector(imgs);
+    Status status;
-    Mat &pano = pano_.getMatRef();
 
-    int64 app_start_time = getTickCount();
+    if ((status = matchImages()) != OK)
+        return status;
+
+    estimateCameraParams();
+
+    if ((status = composePanorama(pano.getMatRef())) != OK)
+        return status;
+
+    LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec");
+    return OK;
+}
 
-    int num_imgs = static_cast<int>(imgs.size());
+
-    if (num_imgs < 2)
+Stitcher::Status Stitcher::matchImages()
+{
+    if ((int)imgs_.size() < 2)
     {
         LOGLN("Need more images");
         return ERR_NEED_MORE_IMGS;
     }
 
-    double work_scale = 1, seam_scale = 1, compose_scale = 1;
+    work_scale_ = 1;
-    bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false;
+    seam_work_aspect_ = 1;
+    seam_scale_ = 1;
+    bool is_work_scale_set = false;
+    bool is_seam_scale_set = false;
+    Mat full_img, img;
+    features_.resize(imgs_.size());
+    seam_est_imgs_.resize(imgs_.size());
+    full_img_sizes_.resize(imgs_.size());
 
     LOGLN("Finding features...");
     int64 t = getTickCount();
 
-    vector<detail::ImageFeatures> features(num_imgs);
+    for (size_t i = 0; i < imgs_.size(); ++i)
-    Mat full_img, img;
-    vector<Mat> seam_est_imgs(num_imgs);
-    vector<Size> full_img_sizes(num_imgs);
-    double seam_work_aspect = 1;
-
-    for (int i = 0; i < num_imgs; ++i)
     {
-        full_img = imgs[i];
+        full_img = imgs_[i];
-        full_img_sizes[i] = full_img.size();
+        full_img_sizes_[i] = full_img.size();
 
         if (registr_resol_ < 0)
         {
             img = full_img;
-            work_scale = 1;
+            work_scale_ = 1;
             is_work_scale_set = true;
         }
         else
         {
             if (!is_work_scale_set)
             {
-                work_scale = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area()));
+                work_scale_ = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area()));
                 is_work_scale_set = true;
             }
-            resize(full_img, img, Size(), work_scale, work_scale);
+            resize(full_img, img, Size(), work_scale_, work_scale_);
         }
         if (!is_seam_scale_set)
         {
-            seam_scale = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area()));
+            seam_scale_ = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area()));
-            seam_work_aspect = seam_scale / work_scale;
+            seam_work_aspect_ = seam_scale_ / work_scale_;
             is_seam_scale_set = true;
         }
 
-        (*features_finder_)(img, features[i]);
+        (*features_finder_)(img, features_[i]);
-        features[i].img_idx = i;
+        features_[i].img_idx = i;
-        LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size());
+        LOGLN("Features in image #" << i+1 << ": " << features_[i].keypoints.size());
 
-        resize(full_img, img, Size(), seam_scale, seam_scale);
+        resize(full_img, img, Size(), seam_scale_, seam_scale_);
-        seam_est_imgs[i] = img.clone();
+        seam_est_imgs_[i] = img.clone();
     }
 
+    // Do it to save memory
     features_finder_->collectGarbage();
     full_img.release();
     img.release();
@@ -152,111 +165,120 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
 
     LOG("Pairwise matching");
     t = getTickCount();
-    vector<detail::MatchesInfo> pairwise_matches;
+    (*features_matcher_)(features_, pairwise_matches_);
-    (*features_matcher_)(features, pairwise_matches);
     features_matcher_->collectGarbage();
     LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
 
     // Leave only images we are sure are from the same panorama
-    vector<int> indices = detail::leaveBiggestComponent(features, pairwise_matches, conf_thresh_);
+    indices_ = detail::leaveBiggestComponent(features_, pairwise_matches_, (float)conf_thresh_);
     vector<Mat> seam_est_imgs_subset;
     vector<Mat> imgs_subset;
     vector<Size> full_img_sizes_subset;
-    for (size_t i = 0; i < indices.size(); ++i)
+    for (size_t i = 0; i < indices_.size(); ++i)
     {
-        imgs_subset.push_back(imgs[indices[i]]);
+        imgs_subset.push_back(imgs_[indices_[i]]);
-        seam_est_imgs_subset.push_back(seam_est_imgs[indices[i]]);
+        seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]);
-        full_img_sizes_subset.push_back(full_img_sizes[indices[i]]);
+        full_img_sizes_subset.push_back(full_img_sizes_[indices_[i]]);
     }
+    seam_est_imgs_ = seam_est_imgs_subset;
+    imgs_ = imgs_subset;
+    full_img_sizes_ = full_img_sizes_subset;
 
-    seam_est_imgs = seam_est_imgs_subset;
+    if ((int)imgs_.size() < 2)
-    imgs = imgs_subset;
-    full_img_sizes = full_img_sizes_subset;
-
-    num_imgs = static_cast<int>(imgs.size());
-    if (num_imgs < 2)
     {
         LOGLN("Need more images");
         return ERR_NEED_MORE_IMGS;
     }
 
-    vector<detail::CameraParams> cameras;
+    return OK;
+}
+
+
+void Stitcher::estimateCameraParams()
+{
     detail::HomographyBasedEstimator estimator;
-    estimator(features, pairwise_matches, cameras);
+    estimator(features_, pairwise_matches_, cameras_);
 
-    for (size_t i = 0; i < cameras.size(); ++i)
+    for (size_t i = 0; i < cameras_.size(); ++i)
     {
         Mat R;
-        cameras[i].R.convertTo(R, CV_32F);
+        cameras_[i].R.convertTo(R, CV_32F);
-        cameras[i].R = R;
+        cameras_[i].R = R;
-        LOGLN("Initial intrinsic parameters #" << indices[i]+1 << ":\n " << cameras[i].K());
+        LOGLN("Initial intrinsic parameters #" << indices_[i] + 1 << ":\n " << cameras_[i].K());
     }
 
     bundle_adjuster_->setConfThresh(conf_thresh_);
-    (*bundle_adjuster_)(features, pairwise_matches, cameras);
+    (*bundle_adjuster_)(features_, pairwise_matches_, cameras_);
 
-    // Find median focal length
+    // Find median focal length and use it as final image scale
     vector<double> focals;
-    for (size_t i = 0; i < cameras.size(); ++i)
+    for (size_t i = 0; i < cameras_.size(); ++i)
     {
-        LOGLN("Camera #" << indices[i]+1 << ":\n" << cameras[i].K());
+        LOGLN("Camera #" << indices_[i] + 1 << ":\n" << cameras_[i].K());
-        focals.push_back(cameras[i].focal);
+        focals.push_back(cameras_[i].focal);
     }
     nth_element(focals.begin(), focals.begin() + focals.size()/2, focals.end());
-    float warped_image_scale = static_cast<float>(focals[focals.size() / 2]);
+    warped_image_scale_ = static_cast<float>(focals[focals.size() / 2]);
 
-    if (horiz_stright_)
+    if (do_wave_correct_)
     {
         vector<Mat> rmats;
-        for (size_t i = 0; i < cameras.size(); ++i)
+        for (size_t i = 0; i < cameras_.size(); ++i)
-            rmats.push_back(cameras[i].R);
+            rmats.push_back(cameras_[i].R);
         detail::waveCorrect(rmats, wave_correct_kind_);
-        for (size_t i = 0; i < cameras.size(); ++i)
+        for (size_t i = 0; i < cameras_.size(); ++i)
-            cameras[i].R = rmats[i];
+            cameras_[i].R = rmats[i];
     }
+}
 
+
+Stitcher::Status Stitcher::composePanorama(Mat &pano)
+{
     LOGLN("Warping images (auxiliary)... ");
-    t = getTickCount();
+    int64 t = getTickCount();
 
-    vector<Point> corners(num_imgs);
+    vector<Point> corners(imgs_.size());
-    vector<Mat> masks_warped(num_imgs);
+    vector<Mat> masks_warped(imgs_.size());
-    vector<Mat> images_warped(num_imgs);
+    vector<Mat> images_warped(imgs_.size());
-    vector<Size> sizes(num_imgs);
+    vector<Size> sizes(imgs_.size());
-    vector<Mat> masks(num_imgs);
+    vector<Mat> masks(imgs_.size());
 
-    // Preapre images masks
+    // Prepare image masks
-    for (int i = 0; i < num_imgs; ++i)
+    for (size_t i = 0; i < imgs_.size(); ++i)
     {
-        masks[i].create(seam_est_imgs[i].size(), CV_8U);
+        masks[i].create(seam_est_imgs_[i].size(), CV_8U);
         masks[i].setTo(Scalar::all(255));
     }
 
     // Warp images and their masks
-    Ptr<detail::Warper> warper = warper_->create(warped_image_scale * seam_work_aspect);
+    Ptr<detail::Warper> warper = warper_->create(float(warped_image_scale_ * seam_work_aspect_));
-    for (int i = 0; i < num_imgs; ++i)
+    for (size_t i = 0; i < imgs_.size(); ++i)
     {
         Mat_<float> K;
-        cameras[i].K().convertTo(K, CV_32F);
+        cameras_[i].K().convertTo(K, CV_32F);
-        K(0,0) *= seam_work_aspect; K(0,2) *= seam_work_aspect;
+        K(0,0) *= (float)seam_work_aspect_; 
-        K(1,1) *= seam_work_aspect; K(1,2) *= seam_work_aspect;
+        K(0,2) *= (float)seam_work_aspect_;
+        K(1,1) *= (float)seam_work_aspect_; 
+        K(1,2) *= (float)seam_work_aspect_;
 
-        corners[i] = warper->warp(seam_est_imgs[i], K, cameras[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT);
+        corners[i] = warper->warp(seam_est_imgs_[i], K, cameras_[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT);
         sizes[i] = images_warped[i].size();
 
-        warper->warp(masks[i], K, cameras[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT);
+        warper->warp(masks[i], K, cameras_[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT);
     }
 
-    vector<Mat> images_warped_f(num_imgs);
+    vector<Mat> images_warped_f(imgs_.size());
-    for (int i = 0; i < num_imgs; ++i)
+    for (size_t i = 0; i < imgs_.size(); ++i)
         images_warped[i].convertTo(images_warped_f[i], CV_32F);
 
     LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
 
+    // Find seams
     exposure_comp_->feed(corners, images_warped, masks_warped);
     seam_finder_->find(images_warped_f, corners, masks_warped);
 
     // Release unused memory
-    seam_est_imgs.clear();
+    seam_est_imgs_.clear();
     images_warped.clear();
     images_warped_f.clear();
     masks.clear();
@@ -266,16 +288,21 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
 
     Mat img_warped, img_warped_s;
     Mat dilated_mask, seam_mask, mask, mask_warped;
+
     double compose_seam_aspect = 1;
     double compose_work_aspect = 1;
     bool is_blender_prepared = false;
 
-    for (int img_idx = 0; img_idx < num_imgs; ++img_idx)
+    double compose_scale = 1;
+    bool is_compose_scale_set = false;
+
+    Mat full_img, img;
+    for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx)
     {
-        LOGLN("Compositing image #" << indices[img_idx]+1);
+        LOGLN("Compositing image #" << indices_[img_idx] + 1);
 
         // Read image and resize it if necessary
-        full_img = imgs[img_idx];
+        full_img = imgs_[img_idx];
         if (!is_compose_scale_set)
         {
             if (compose_resol_ > 0)
@@ -283,32 +310,32 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
             is_compose_scale_set = true;
 
             // Compute relative scales
-            compose_seam_aspect = compose_scale / seam_scale;
+            compose_seam_aspect = compose_scale / seam_scale_;
-            compose_work_aspect = compose_scale / work_scale;
+            compose_work_aspect = compose_scale / work_scale_;
 
             // Update warped image scale
-            warped_image_scale *= static_cast<float>(compose_work_aspect);
+            warped_image_scale_ *= static_cast<float>(compose_work_aspect);
-            warper = warper_->create(warped_image_scale);
+            warper = warper_->create((float)warped_image_scale_);
 
             // Update corners and sizes
-            for (int i = 0; i < num_imgs; ++i)
+            for (size_t i = 0; i < imgs_.size(); ++i)
             {
                 // Update intrinsics
-                cameras[i].focal *= compose_work_aspect;
+                cameras_[i].focal *= compose_work_aspect;
-                cameras[i].ppx *= compose_work_aspect;
+                cameras_[i].ppx *= compose_work_aspect;
-                cameras[i].ppy *= compose_work_aspect;
+                cameras_[i].ppy *= compose_work_aspect;
 
                 // Update corner and size
-                Size sz = full_img_sizes[i];
+                Size sz = full_img_sizes_[i];
                 if (std::abs(compose_scale - 1) > 1e-1)
                 {
-                    sz.width = cvRound(full_img_sizes[i].width * compose_scale);
+                    sz.width = cvRound(full_img_sizes_[i].width * compose_scale);
-                    sz.height = cvRound(full_img_sizes[i].height * compose_scale);
+                    sz.height = cvRound(full_img_sizes_[i].height * compose_scale);
                 }
 
                 Mat K;
-                cameras[i].K().convertTo(K, CV_32F);
+                cameras_[i].K().convertTo(K, CV_32F);
-                Rect roi = warper->warpRoi(sz, K, cameras[i].R);
+                Rect roi = warper->warpRoi(sz, K, cameras_[i].R);
                 corners[i] = roi.tl();
                 sizes[i] = roi.size();
             }
@@ -321,15 +348,15 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
         Size img_size = img.size();
 
         Mat K;
-        cameras[img_idx].K().convertTo(K, CV_32F);
+        cameras_[img_idx].K().convertTo(K, CV_32F);
 
         // Warp the current image
-        warper->warp(img, K, cameras[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT);
+        warper->warp(img, K, cameras_[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT);
 
         // Warp the current image mask
         mask.create(img_size, CV_8U);
         mask.setTo(Scalar::all(255));
-        warper->warp(mask, K, cameras[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT);
+        warper->warp(mask, K, cameras_[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT);
 
         // Compensate exposure
         exposure_comp_->apply(img_idx, corners[img_idx], img_warped, mask_warped);
@@ -339,8 +366,10 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
         img.release();
         mask.release();
 
+        // Make sure seam mask has proper size
         dilate(masks_warped[img_idx], dilated_mask, Mat());
         resize(dilated_mask, seam_mask, mask_warped.size());
+
         mask_warped = seam_mask & mask_warped;
 
         if (!is_blender_prepared)
@@ -362,8 +391,6 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
     // so convert it to avoid user confusing
     result.convertTo(pano, CV_8U);
 
-    LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec");
-
     return OK;
 }