Merge pull request #1532 from catree:add_gms_matcher

modules/xfeatures2d/doc/xfeatures2d.bib

@@ -122,3 +122,10 @@
   year={2004},
   publisher={Springer}
 }
+
+@inproceedings{Bian2017gms,
+  title={GMS: Grid-based Motion Statistics for Fast, Ultra-robust Feature Correspondence},
+  author={JiaWang Bian and Wen-Yan Lin and Yasuyuki Matsushita and Sai-Kit Yeung and Tan Dat Nguyen and Ming-Ming Cheng},
+  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
+  year={2017}
+}

modules/xfeatures2d/include/opencv2/xfeatures2d.hpp

@@ -53,6 +53,10 @@ This section describes experimental algorithms for 2d feature detection.
 This section describes two popular algorithms for 2d feature detection, SIFT and SURF, that are
 known to be patented. Use them at your own risk.
 
+    @defgroup xfeatures2d_match Experimental 2D Features Matching Algorithm
+
+This section describes the GMS (Grid-based Motion Statistics) matching strategy.
+
 @}
 */
 
@@ -962,6 +966,32 @@ CV_EXPORTS void FASTForPointSet( InputArray image, CV_IN_OUT std::vector<KeyPoin
 
 //! @}
 
+
+//! @addtogroup xfeatures2d_match
+//! @{
+
+/** @brief GMS  (Grid-based Motion Statistics) feature matching strategy by @cite Bian2017gms .
+    @param size1 Input size of image1.
+    @param size2 Input size of image2.
+    @param keypoints1 Input keypoints of image1.
+    @param keypoints2 Input keypoints of image2.
+    @param matches1to2 Input 1-nearest neighbor matches.
+    @param matchesGMS Matches returned by the GMS matching strategy.
+    @param withRotation Take rotation transformation into account.
+    @param withScale Take scale transformation into account.
+    @param thresholdFactor The higher, the less matches.
+    @note
+        Since GMS works well when the number of features is large, we recommend to use the ORB feature and set FastThreshold to 0 to get as many as possible features quickly.
+        If matching results are not satisfying, please add more features. (We use 10000 for images with 640 X 480).
+        If your images have big rotation and scale changes, please set withRotation or withScale to true.
+ */
+
+CV_EXPORTS void matchGMS( const Size& size1, const Size& size2, const std::vector<KeyPoint>& keypoints1, const std::vector<KeyPoint>& keypoints2,
+                          const std::vector<DMatch>& matches1to2, std::vector<DMatch>& matchesGMS, const bool withRotation = false,
+                          const bool withScale = false, const double thresholdFactor = 6.0 );
+
+//! @}
+
 }
 }
 

modules/xfeatures2d/samples/gms_matcher.cpp

+#include <iostream>
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/features2d.hpp>
+#include <opencv2/flann.hpp>
+#include <opencv2/xfeatures2d.hpp>
+
+using namespace cv;
+using namespace cv::xfeatures2d;
+
+////////////////////////////////////////////////////
+// This program demonstrates the GMS matching strategy.
+int main(int argc, char* argv[])
+{
+    const char* keys =
+        "{ h help        |                  | print help message  }"
+        "{ l left        |                  | specify left (reference) image  }"
+        "{ r right       |                  | specify right (query) image }"
+        "{ camera        | 0                | specify the camera device number }"
+        "{ nfeatures     | 10000            | specify the maximum number of ORB features }"
+        "{ fastThreshold | 20               | specify the FAST threshold }"
+        "{ drawSimple    | true             | do not draw not matched keypoints }"
+        "{ withRotation  | false            | take rotation into account }"
+        "{ withScale     | false            | take scale into account }";
+
+    CommandLineParser cmd(argc, argv, keys);
+    if (cmd.has("help"))
+    {
+        std::cout << "Usage: gms_matcher [options]" << std::endl;
+        std::cout << "Available options:" << std::endl;
+        cmd.printMessage();
+        return EXIT_SUCCESS;
+    }
+
+    Ptr<Feature2D> orb = ORB::create(cmd.get<int>("nfeatures"));
+    orb.dynamicCast<cv::ORB>()->setFastThreshold(cmd.get<int>("fastThreshold"));
+    Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-Hamming");
+
+    if (!cmd.get<String>("left").empty() && !cmd.get<String>("right").empty())
+    {
+        Mat imgL = imread(cmd.get<String>("left"));
+        Mat imgR = imread(cmd.get<String>("right"));
+
+        std::vector<KeyPoint> kpRef, kpCur;
+        Mat descRef, descCur;
+        orb->detectAndCompute(imgL, noArray(), kpRef, descRef);
+        orb->detectAndCompute(imgR, noArray(), kpCur, descCur);
+
+        std::vector<DMatch> matchesAll, matchesGMS;
+        matcher->match(descCur, descRef, matchesAll);
+
+        matchGMS(imgR.size(), imgL.size(), kpCur, kpRef, matchesAll, matchesGMS, cmd.get<bool>("withRotation"), cmd.get<bool>("withScale"));
+        std::cout << "matchesGMS: " << matchesGMS.size() << std::endl;
+
+        Mat frameMatches;
+        if (cmd.get<bool>("drawSimple"))
+            drawMatches(imgR, kpCur, imgL, kpRef, matchesGMS, frameMatches, Scalar::all(-1), Scalar::all(-1),
+                        std::vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
+        else
+            drawMatches(imgR, kpCur, imgL, kpRef, matchesGMS, frameMatches);
+        imshow("Matches GMS", frameMatches);
+        waitKey();
+    }
+    else
+    {
+        std::vector<KeyPoint> kpRef;
+        Mat descRef;
+
+        VideoCapture capture(cmd.get<int>("camera"));
+        //Camera warm-up
+        for (int i = 0; i < 10; i++)
+        {
+            Mat frame;
+            capture >> frame;
+        }
+
+        Mat frameRef;
+        for (;;)
+        {
+            Mat frame;
+            capture >> frame;
+
+            if (frameRef.empty())
+            {
+                frame.copyTo(frameRef);
+                orb->detectAndCompute(frameRef, noArray(), kpRef, descRef);
+            }
+
+            TickMeter tm;
+            tm.start();
+            std::vector<KeyPoint> kp;
+            Mat desc;
+            orb->detectAndCompute(frame, noArray(), kp, desc);
+            tm.stop();
+            double t_orb = tm.getTimeMilli();
+
+            tm.reset();
+            tm.start();
+            std::vector<DMatch> matchesAll, matchesGMS;
+            matcher->match(desc, descRef, matchesAll);
+            tm.stop();
+            double t_match = tm.getTimeMilli();
+
+            matchGMS(frame.size(), frameRef.size(), kp, kpRef, matchesAll, matchesGMS, cmd.get<bool>("withRotation"), cmd.get<bool>("withScale"));
+            tm.stop();
+            Mat frameMatches;
+            if (cmd.get<bool>("drawSimple"))
+                 drawMatches(frame, kp, frameRef, kpRef, matchesGMS, frameMatches, Scalar::all(-1), Scalar::all(-1),
+                            std::vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
+            else
+                 drawMatches(frame, kp, frameRef, kpRef, matchesGMS, frameMatches);
+
+            String label = format("ORB: %.2f ms", t_orb);
+            putText(frameMatches, label, Point(20, 20), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255));
+            label = format("Matching: %.2f ms", t_match);
+            putText(frameMatches, label, Point(20, 40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255));
+            label = format("GMS matching: %.2f ms", tm.getTimeMilli());
+            putText(frameMatches, label, Point(20, 60), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255));
+            putText(frameMatches, "Press r to reinitialize the reference image.", Point(frameMatches.cols-380, 20), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255));
+            putText(frameMatches, "Press esc to quit.", Point(frameMatches.cols-180, 40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255));
+
+            imshow("Matches GMS", frameMatches);
+            int c = waitKey(30);
+            if (c == 27)
+                break;
+            else if (c == 'r')
+            {
+                frame.copyTo(frameRef);
+                orb->detectAndCompute(frameRef, noArray(), kpRef, descRef);
+            }
+        }
+
+    }
+
+    return EXIT_SUCCESS;
+}

modules/xfeatures2d/test/test_gms_matcher.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "test_precomp.hpp"
+
+using namespace std;
+using namespace cv;
+using namespace cv::xfeatures2d;
+
+class CV_GMSMatcherTest : public cvtest::BaseTest
+{
+public:
+    CV_GMSMatcherTest();
+    ~CV_GMSMatcherTest();
+
+protected:
+    virtual void run(int);
+
+    bool combinations[4][2];
+    double eps[3][4]; //3 imgs x 4 combinations
+    double correctMatchDistThreshold;
+};
+
+CV_GMSMatcherTest::CV_GMSMatcherTest()
+{
+    combinations[0][0] = false; combinations[0][1] = false;
+    combinations[1][0] = false; combinations[1][1] = true;
+    combinations[2][0] = true; combinations[2][1] = false;
+    combinations[3][0] = true; combinations[3][1] = true;
+
+    //Threshold = truncate(min(acc_win32, acc_win64))
+    eps[0][0] = 0.9313;
+    eps[0][1] = 0.9223;
+    eps[0][2] = 0.9313;
+    eps[0][3] = 0.9223;
+
+    eps[1][0] = 0.8199;
+    eps[1][1] = 0.7964;
+    eps[1][2] = 0.8199;
+    eps[1][3] = 0.7964;
+
+    eps[2][0] = 0.7098;
+    eps[2][1] = 0.6659;
+    eps[2][2] = 0.6939;
+    eps[2][3] = 0.6457;
+
+    correctMatchDistThreshold = 5.0;
+}
+
+CV_GMSMatcherTest::~CV_GMSMatcherTest() {}
+
+void CV_GMSMatcherTest::run( int )
+{
+    ts->set_failed_test_info(cvtest::TS::OK);
+
+    Mat imgRef = imread(string(ts->get_data_path()) + "detectors_descriptors_evaluation/images_datasets/graf/img1.png");
+
+    Ptr<Feature2D> orb = ORB::create(10000);
+    vector<KeyPoint> keypointsRef, keypointsCur;
+    Mat descriptorsRef, descriptorsCur;
+    orb->detectAndCompute(imgRef, noArray(), keypointsRef, descriptorsRef);
+
+    vector<DMatch> matchesAll, matchesGMS;
+    Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-Hamming");
+
+    const int startImg = 2;
+    const int nImgs = 3;
+    for (int num = startImg; num < startImg+nImgs; num++)
+    {
+        string imgPath = string(ts->get_data_path()) + format("detectors_descriptors_evaluation/images_datasets/graf/img%d.png", num);
+        Mat imgCur = imread(imgPath);
+        orb->detectAndCompute(imgCur, noArray(), keypointsCur, descriptorsCur);
+
+        matcher->match(descriptorsCur, descriptorsRef, matchesAll);
+
+        string xml = string(ts->get_data_path()) + format("detectors_descriptors_evaluation/images_datasets/graf/H1to%dp.xml", num);
+        FileStorage fs(xml, FileStorage::READ);
+        if (!fs.isOpened())
+        {
+            ts->set_failed_test_info(cvtest::TS::FAIL_INVALID_TEST_DATA);
+            return;
+        }
+
+        Mat H1toCur;
+        fs[format("H1%d", num)] >> H1toCur;
+
+        for (int comb = 0; comb < 4; comb++)
+        {
+            matchGMS(imgCur.size(), imgRef.size(), keypointsCur, keypointsRef, matchesAll, matchesGMS, combinations[comb][0], combinations[comb][1]);
+
+            int nbCorrectMatches = 0;
+            for (size_t i = 0; i < matchesGMS.size(); i++)
+            {
+                Point2f ptRef = keypointsRef[matchesGMS[i].trainIdx].pt;
+                Point2f ptCur = keypointsCur[matchesGMS[i].queryIdx].pt;
+                Mat matRef = (Mat_<double>(3,1) << ptRef.x, ptRef.y, 1);
+                Mat matTrans = H1toCur * matRef;
+                Point2f ptTrans( (float) (matTrans.at<double>(0,0)/matTrans.at<double>(2,0)),
+                                 (float) (matTrans.at<double>(1,0)/matTrans.at<double>(2,0)));
+
+                if (norm(ptTrans-ptCur) < correctMatchDistThreshold)
+                    nbCorrectMatches++;
+            }
+
+            double ratio = nbCorrectMatches / (double) matchesGMS.size();
+            if (ratio < eps[num-startImg][comb])
+            {
+                ts->printf( cvtest::TS::LOG, "Invalid accuracy for image %s and combination withRotation=%d withScale=%d, "
+                                             "matches ratio is %f, ratio threshold is %f, distance threshold is %f.\n",
+                            imgPath.substr(imgPath.size()-8).c_str(), combinations[comb][0], combinations[comb][1], ratio,
+                            eps[num-startImg][comb], correctMatchDistThreshold);
+                ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY);
+            }
+        }
+    }
+}
+
+TEST(XFeatures2d_GMSMatcher, gms_matcher_regression) { CV_GMSMatcherTest test; test.safe_run(); }