added leaveBiggestComponent function into opencv_stitching

modules/stitching/main.cpp

@@ -15,7 +15,7 @@ void printUsage()
     cout << "Usage: opencv_stitching img1 img2 [...imgN]\n" 
         << "\t[--matchconf <float>]\n"
         << "\t[--ba (ray|focal_ray)]\n"
-        << "\t[--ba_thresh <float>]\n"
+        << "\t[--conf_thresh <float>]\n"
         << "\t[--wavecorrect (no|yes)]\n"
         << "\t[--warp (plane|cylindrical|spherical)]\n" 
         << "\t[--seam (no|voronoi|graphcut)]\n" 
@@ -23,7 +23,7 @@ void printUsage()
         << "\t[--output <result_img>]\n\n";
     cout << "--matchconf\n"
         << "\tGood values are in [0.2, 0.8] range usually.\n\n";
-    cout << "--ba_thresh\n"
+    cout << "--conf_thresh\n"
         << "\tGood values are in [0.3, 1.0] range usually.\n";
 }
 
@@ -34,7 +34,7 @@ int main(int argc, char* argv[])
     vector<Mat> images;
     string result_name = "result.png";
     int ba_space = BundleAdjuster::FOCAL_RAY_SPACE;
-    float ba_thresh = 1.f;
+    float conf_thresh = 1.f;
     bool wave_correct = true;
     int warp_type = Warper::SPHERICAL;
     bool user_match_conf = false;
@@ -74,9 +74,9 @@ int main(int argc, char* argv[])
             }
             i++;
         }
-        else if (string(argv[i]) == "--ba_thresh")
+        else if (string(argv[i]) == "--conf_thresh")
         {
-            ba_thresh = static_cast<float>(atof(argv[i + 1]));
+            conf_thresh = static_cast<float>(atof(argv[i + 1]));
             i++;
         }
         else if (string(argv[i]) == "--wavecorrect")
@@ -154,7 +154,7 @@ int main(int argc, char* argv[])
         }
     }
 
-    const int num_images = static_cast<int>(images.size());
+    int num_images = static_cast<int>(images.size());
     if (num_images < 2)
     {
         cout << "Need more images\n";
@@ -173,6 +173,9 @@ int main(int argc, char* argv[])
         matcher = BestOf2NearestMatcher(true, match_conf);
     matcher(images, features, pairwise_matches);
 
+    leaveBiggestComponent(images, features, pairwise_matches, conf_thresh);
+    num_images = static_cast<int>(images.size());
+
     LOGLN("Estimating rotations...");
     HomographyBasedEstimator estimator;
     vector<CameraParams> cameras;
@@ -187,7 +190,7 @@ int main(int argc, char* argv[])
     }
 
     LOGLN("Bundle adjustment...");
-    BundleAdjuster adjuster(ba_space, ba_thresh);
+    BundleAdjuster adjuster(ba_space, conf_thresh);
     adjuster(images, features, pairwise_matches, cameras);
 
     if (wave_correct)

modules/stitching/matchers.cpp

@@ -123,22 +123,19 @@ void SurfFeaturesFinder::find(const vector<Mat> &images, vector<ImageFeatures> &
 
 //////////////////////////////////////////////////////////////////////////////
 
-MatchesInfo::MatchesInfo() : src_img_idx(-1), dst_img_idx(-1), num_inliers(0) {}
-
-
-MatchesInfo::MatchesInfo(const MatchesInfo &other)
-{
-    *this = other;
-}
+MatchesInfo::MatchesInfo() : src_img_idx(-1), dst_img_idx(-1), num_inliers(0), confidence(0) {}
 
+MatchesInfo::MatchesInfo(const MatchesInfo &other) { *this = other; }
 
 const MatchesInfo& MatchesInfo::operator =(const MatchesInfo &other)
 {
     src_img_idx = other.src_img_idx;
     dst_img_idx = other.dst_img_idx;
     matches = other.matches;
+    inliers_mask = other.inliers_mask;
     num_inliers = other.num_inliers;
     H = other.H.clone();
+    confidence = other.confidence;
     return *this;
 }
 
@@ -303,14 +300,14 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features
     Mat dst_points(1, matches_info.matches.size(), CV_32FC2);
     for (size_t i = 0; i < matches_info.matches.size(); ++i)
     {
-        const DMatch& r = matches_info.matches[i];
+        const DMatch& m = matches_info.matches[i];
 
-        Point2f p = features1.keypoints[r.queryIdx].pt;
+        Point2f p = features1.keypoints[m.queryIdx].pt;
         p.x -= img1.cols * 0.5f;
         p.y -= img1.rows * 0.5f;
         src_points.at<Point2f>(0, i) = p;
 
-        p = features2.keypoints[r.trainIdx].pt;
+        p = features2.keypoints[m.trainIdx].pt;
         p.x -= img2.cols * 0.5f;
         p.y -= img2.rows * 0.5f;
         dst_points.at<Point2f>(0, i) = p;
@@ -325,6 +322,8 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features
         if (matches_info.inliers_mask[i])
             matches_info.num_inliers++;
 
+    matches_info.confidence = matches_info.num_inliers / (8 + 0.3*matches_info.matches.size());
+
     // Check if we should try to refine motion
     if (matches_info.num_inliers < num_matches_thresh2_)
         return;
@@ -338,14 +337,14 @@ void BestOf2NearestMatcher::match(const Mat &img1, const ImageFeatures &features
         if (!matches_info.inliers_mask[i])
             continue;
 
-        const DMatch& r = matches_info.matches[i];
+        const DMatch& m = matches_info.matches[i];
 
-        Point2f p = features1.keypoints[r.queryIdx].pt;
+        Point2f p = features1.keypoints[m.queryIdx].pt;
         p.x -= img1.cols * 0.5f;
         p.y -= img2.rows * 0.5f;
         src_points.at<Point2f>(0, inlier_idx) = p;
 
-        p = features2.keypoints[r.trainIdx].pt;
+        p = features2.keypoints[m.trainIdx].pt;
         p.x -= img2.cols * 0.5f;
         p.y -= img2.rows * 0.5f;
         dst_points.at<Point2f>(0, inlier_idx) = p;

modules/stitching/matchers.hpp

@@ -43,11 +43,12 @@ struct MatchesInfo
     MatchesInfo(const MatchesInfo &other);
     const MatchesInfo& operator =(const MatchesInfo &other);
 
-    int src_img_idx, dst_img_idx; // Optional images indices
+    int src_img_idx, dst_img_idx;       // Images indices (optional)
     std::vector<cv::DMatch> matches;
-    std::vector<uchar> inliers_mask;
-    int num_inliers; // Number of geometrically consistent matches
-    cv::Mat H; // Homography
+    std::vector<uchar> inliers_mask;    // Geometrically consistent matches mask
+    int num_inliers;                    // Number of geometrically consistent matches
+    cv::Mat H;                          // Estimated homography
+    double confidence;                  // Confidence two images are from the same panorama
 };
 
 

modules/stitching/motion_estimators.cpp

@@ -13,12 +13,7 @@ using namespace cv;
 
 CameraParams::CameraParams() : focal(1), R(Mat::eye(3, 3, CV_64F)), t(Mat::zeros(3, 1, CV_64F)) {}
 
-
-CameraParams::CameraParams(const CameraParams &other)
-{
-    *this = other;
-}
-
+CameraParams::CameraParams(const CameraParams &other) { *this = other; }
 
 const CameraParams& CameraParams::operator =(const CameraParams &other)
 {
@@ -84,13 +79,14 @@ void HomographyBasedEstimator::estimate(const vector<Mat> &images, const vector<
             int pair_idx = i * num_images + j;
             if (pairwise_matches[pair_idx].H.empty())
                 continue;
+
             double f_to, f_from;
             bool f_to_ok, f_from_ok;
             focalsFromHomography(pairwise_matches[pair_idx].H.inv(), f_to, f_from, f_to_ok, f_from_ok);
-            if (f_from_ok)
-                focals.push_back(f_from);
-            if (f_to_ok)
-                focals.push_back(f_to);
+
+            if (f_from_ok) focals.push_back(f_from);
+            if (f_to_ok) focals.push_back(f_to);
+
             if (f_from_ok && f_to_ok)
             {
                 is_focal_estimated[i] = true;
@@ -98,6 +94,7 @@ void HomographyBasedEstimator::estimate(const vector<Mat> &images, const vector<
             }
         }
     }
+
     is_focals_estimated_ = true;
     for (int i = 0; i < num_images; ++i)
         is_focals_estimated_ = is_focals_estimated_ && is_focal_estimated[i];
@@ -132,9 +129,12 @@ void BundleAdjuster::estimate(const vector<Mat> &images, const vector<ImageFeatu
     for (int i = 0; i < num_images_; ++i)
     {
         cameras_.at<double>(i * 4, 0) = cameras[i].focal;
+
         svd(cameras[i].R, SVD::FULL_UV);
         Mat R = svd.u * svd.vt;
-        if (determinant(R) < 0) R *= -1;
+        if (determinant(R) < 0) 
+            R *= -1;
+
         Mat rvec;
         Rodrigues(R, rvec); CV_Assert(rvec.type() == CV_32F);
         cameras_.at<double>(i * 4 + 1, 0) = rvec.at<float>(0, 0);
@@ -142,20 +142,19 @@ void BundleAdjuster::estimate(const vector<Mat> &images, const vector<ImageFeatu
         cameras_.at<double>(i * 4 + 3, 0) = rvec.at<float>(2, 0);
     }
 
+    // Select only consistent image pairs for futher adjustment
     edges_.clear();
     for (int i = 0; i < num_images_ - 1; ++i)
     {
         for (int j = i + 1; j < num_images_; ++j)
-        {
-            int pair_idx = i * num_images_ + j;
-            const MatchesInfo& mi = pairwise_matches_[pair_idx];
-            float ni = static_cast<float>(mi.num_inliers);
-            float nf = static_cast<float>(mi.matches.size());
-            if (ni / (8.f + 0.3f * nf) > dist_thresh_)
+        {
+            const MatchesInfo& matches_info = pairwise_matches_[i * num_images_ + j];
+            if (matches_info.confidence > conf_thresh_)
                 edges_.push_back(make_pair(i, j));
         }
     }
 
+    // Compute number of correspondences
     total_num_matches_ = 0;
     for (size_t i = 0; i < edges_.size(); ++i)
         total_num_matches_ += static_cast<int>(pairwise_matches[edges_[i].first * num_images_ + edges_[i].second].num_inliers);
@@ -369,7 +368,6 @@ void waveCorrect(vector<Mat> &rmats)
     normalize(r0, r0);
 
     r1.cross(r0).copyTo(r2);
-
     if (determinant(R) < 0)
         R *= -1;
 
@@ -380,6 +378,64 @@ void waveCorrect(vector<Mat> &rmats)
 
 //////////////////////////////////////////////////////////////////////////////
 
+void leaveBiggestComponent(vector<Mat> &images, vector<ImageFeatures> &features, 
+                           vector<MatchesInfo> &pairwise_matches, float conf_threshold)
+{
+    const int num_images = static_cast<int>(images.size());
+
+    DjSets comps(num_images);
+    for (int i = 0; i < num_images; ++i)
+    {
+        for (int j = 0; j < num_images; ++j)
+        {
+            if (pairwise_matches[i*num_images + j].confidence < conf_threshold)
+                continue;
+            int comp1 = comps.find(i);
+            int comp2 = comps.find(j);
+            if (comp1 != comp2) 
+                comps.merge(comp1, comp2);
+        }
+    }
+
+    int max_comp = max_element(comps.size.begin(), comps.size.end()) - comps.size.begin();
+
+    vector<int> indices;
+    vector<int> indices_removed;
+    for (int i = 0; i < num_images; ++i)
+        if (comps.find(i) == max_comp)
+            indices.push_back(i);    
+        else
+            indices_removed.push_back(i);
+
+    vector<Mat> images_subset;
+    vector<ImageFeatures> features_subset;
+    vector<MatchesInfo> pairwise_matches_subset;
+    for (size_t i = 0; i < indices.size(); ++i)
+    {
+        images_subset.push_back(images[indices[i]]);
+        features_subset.push_back(features[indices[i]]);
+        for (size_t j = 0; j < indices.size(); ++j)
+        {
+            pairwise_matches_subset.push_back(pairwise_matches[indices[i]*num_images + indices[j]]);
+            pairwise_matches_subset.back().src_img_idx = i;
+            pairwise_matches_subset.back().dst_img_idx = j;
+        }
+    }
+
+    if (static_cast<int>(images_subset.size()) == num_images)
+        return;
+
+    LOG("Removed some images, because can't match them: (");
+    LOG(indices_removed[0]);
+    for (size_t i = 1; i < indices_removed.size(); ++i) LOG(", " << indices_removed[i]);
+    LOGLN(")");
+
+    images = images_subset;
+    features = features_subset;
+    pairwise_matches = pairwise_matches_subset;
+}
+
+
 void findMaxSpanningTree(int num_images, const vector<MatchesInfo> &pairwise_matches,
                          Graph &span_tree, vector<int> &centers)
 {
@@ -391,6 +447,8 @@ void findMaxSpanningTree(int num_images, const vector<MatchesInfo> &pairwise_mat
     {
         for (int j = 0; j < num_images; ++j)
         {
+            if (pairwise_matches[i * num_images + j].H.empty())
+                continue;
             float conf = static_cast<float>(pairwise_matches[i * num_images + j].num_inliers);
             graph.addEdge(i, j, conf);
             edges.push_back(GraphEdge(i, j, conf));

modules/stitching/motion_estimators.hpp

@@ -52,8 +52,8 @@ class BundleAdjuster : public Estimator
 public:
     enum { RAY_SPACE, FOCAL_RAY_SPACE };
 
-    BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float dist_thresh = 1.f) 
-        : cost_space_(cost_space), dist_thresh_(dist_thresh) {}
+    BundleAdjuster(int cost_space = FOCAL_RAY_SPACE, float conf_thresh = 1.f) 
+        : cost_space_(cost_space), conf_thresh_(conf_thresh) {}
 
 private:
     void estimate(const std::vector<cv::Mat> &images, const std::vector<ImageFeatures> &features,
@@ -71,7 +71,7 @@ private:
     std::vector<std::pair<int,int> > edges_;
 
     int cost_space_;
-    float dist_thresh_;
+    float conf_thresh_;
     cv::Mat err_, err1_, err2_;
     cv::Mat J_;
 };
@@ -83,6 +83,9 @@ void waveCorrect(std::vector<cv::Mat> &rmats);
 //////////////////////////////////////////////////////////////////////////////
 // Auxiliary functions
 
+void leaveBiggestComponent(std::vector<cv::Mat> &images, std::vector<ImageFeatures> &features, 
+                           std::vector<MatchesInfo> &pairwise_matches, float conf_threshold);
+
 void findMaxSpanningTree(int num_images, const std::vector<MatchesInfo> &pairwise_matches, 
                          Graph &span_tree, std::vector<int> &centers);