Added support of homography estimation into videostab module

modules/videostab/include/opencv2/videostab/global_motion.hpp

@@ -58,7 +58,9 @@ enum MotionModel
     TRANSLATION = 0,
     TRANSLATION_AND_SCALE = 1,
     LINEAR_SIMILARITY = 2,
-    AFFINE = 3
+    AFFINE = 3,
+    HOMOGRAPHY = 4,
+    UNKNOWN = 5
 };
 
 CV_EXPORTS Mat estimateGlobalMotionLeastSquares(
@@ -76,10 +78,11 @@ struct CV_EXPORTS RansacParams
     RansacParams(int size, float thresh, float eps, float prob)
         : size(size), thresh(thresh), eps(eps), prob(prob) {}
 
-    static RansacParams translationMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); }
+    static RansacParams translation2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); }
     static RansacParams translationAndScale2dMotionStd() { return RansacParams(3, 0.5f, 0.5f, 0.99f); }
-    static RansacParams linearSimilarityMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); }
+    static RansacParams linearSimilarity2dMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); }
     static RansacParams affine2dMotionStd() { return RansacParams(6, 0.5f, 0.5f, 0.99f); }
+    static RansacParams homography2dMotionStd() { return RansacParams(8, 0.5f, 0.5f, 0.99f); }
 };
 
 CV_EXPORTS Mat estimateGlobalMotionRobust(
@@ -87,14 +90,22 @@ CV_EXPORTS Mat estimateGlobalMotionRobust(
         int model = AFFINE, const RansacParams &params = RansacParams::affine2dMotionStd(),
         float *rmse = 0, int *ninliers = 0);
 
-class CV_EXPORTS IGlobalMotionEstimator
+class CV_EXPORTS GlobalMotionEstimatorBase
 {
 public:
-    virtual ~IGlobalMotionEstimator() {}
+    GlobalMotionEstimatorBase() : motionModel_(UNKNOWN) {}
+    virtual ~GlobalMotionEstimatorBase() {}
+
+    virtual void setMotionModel(MotionModel val) { motionModel_ = val; }
+    virtual MotionModel motionModel() const { return motionModel_; }
+
     virtual Mat estimate(const Mat &frame0, const Mat &frame1) = 0;
+
+protected:
+    MotionModel motionModel_;
 };
 
-class CV_EXPORTS PyrLkRobustMotionEstimator : public IGlobalMotionEstimator
+class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase
 {
 public:
     PyrLkRobustMotionEstimator();
@@ -105,9 +116,6 @@ public:
     void setOptFlowEstimator(Ptr<ISparseOptFlowEstimator> val) { optFlowEstimator_ = val; }
     Ptr<ISparseOptFlowEstimator> optFlowEstimator() const { return optFlowEstimator_; }
 
-    void setMotionModel(MotionModel val) { motionModel_ = val; }
-    MotionModel motionModel() const { return motionModel_; }
-
     void setRansacParams(const RansacParams &val) { ransacParams_ = val; }
     RansacParams ransacParams() const { return ransacParams_; }
 
@@ -122,7 +130,6 @@ public:
 private:
     Ptr<FeatureDetector> detector_;
     Ptr<ISparseOptFlowEstimator> optFlowEstimator_;
-    MotionModel motionModel_;
     RansacParams ransacParams_;
     std::vector<uchar> status_;
     std::vector<KeyPoint> keypointsPrev_;

modules/videostab/include/opencv2/videostab/inpainting.hpp

@@ -47,6 +47,7 @@
 #include "opencv2/core/core.hpp"
 #include "opencv2/videostab/optical_flow.hpp"
 #include "opencv2/videostab/fast_marching.hpp"
+#include "opencv2/videostab/global_motion.hpp"
 #include "opencv2/photo/photo.hpp"
 
 namespace cv
@@ -66,6 +67,9 @@ public:
     virtual void setRadius(int val) { radius_ = val; }
     virtual int radius() const { return radius_; }
 
+    virtual void setMotionModel(MotionModel val) { motionModel_ = val; }
+    virtual MotionModel motionModel() const { return motionModel_; }
+
     virtual void setFrames(const std::vector<Mat> &val) { frames_ = &val; }
     virtual const std::vector<Mat>& frames() const { return *frames_; }
 
@@ -82,6 +86,7 @@ public:
 
 protected:
     int radius_;
+    MotionModel motionModel_;
     const std::vector<Mat> *frames_;
     const std::vector<Mat> *motions_;
     const std::vector<Mat> *stabilizedFrames_;
@@ -101,6 +106,7 @@ public:
     bool empty() const { return inpainters_.empty(); }
 
     virtual void setRadius(int val);
+    virtual void setMotionModel(MotionModel val);
     virtual void setFrames(const std::vector<Mat> &val);
     virtual void setMotions(const std::vector<Mat> &val);
     virtual void setStabilizedFrames(const std::vector<Mat> &val);

modules/videostab/include/opencv2/videostab/stabilizer.hpp

@@ -72,8 +72,8 @@ public:
     void setFrameSource(Ptr<IFrameSource> val) { frameSource_ = val; }
     Ptr<IFrameSource> frameSource() const { return frameSource_; }
 
-    void setMotionEstimator(Ptr<IGlobalMotionEstimator> val) { motionEstimator_ = val; }
-    Ptr<IGlobalMotionEstimator> motionEstimator() const { return motionEstimator_; }
+    void setMotionEstimator(Ptr<GlobalMotionEstimatorBase> val) { motionEstimator_ = val; }
+    Ptr<GlobalMotionEstimatorBase> motionEstimator() const { return motionEstimator_; }
 
     void setDeblurer(Ptr<DeblurerBase> val) { deblurer_ = val; }
     Ptr<DeblurerBase> deblurrer() const { return deblurer_; }
@@ -104,7 +104,7 @@ protected:
 
     Ptr<ILog> log_;
     Ptr<IFrameSource> frameSource_;
-    Ptr<IGlobalMotionEstimator> motionEstimator_;
+    Ptr<GlobalMotionEstimatorBase> motionEstimator_;
     Ptr<DeblurerBase> deblurer_;
     Ptr<InpainterBase> inpainter_;
     int radius_;

modules/videostab/src/global_motion.cpp

@@ -41,7 +41,6 @@
 //M*/
 
 #include "precomp.hpp"
-#include "opencv2/highgui/highgui.hpp"
 #include "opencv2/videostab/global_motion.hpp"
 #include "opencv2/videostab/ring_buffer.hpp"
 
@@ -324,8 +323,38 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1)
 
     float rmse;
     int ninliers;
-    Mat M = estimateGlobalMotionRobust(
-            pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, &rmse, &ninliers);
+    Mat_<float> M;
+
+    if (motionModel_ != HOMOGRAPHY)
+        M = estimateGlobalMotionRobust(
+                pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, &rmse, &ninliers);
+    else
+    {
+        vector<uchar> mask;
+        M = findHomography(pointsPrevGood_, pointsGood_, CV_RANSAC, ransacParams_.thresh, mask);
+
+        ninliers = 0;
+        rmse = 0;
+
+        Point2d p0, p1;
+        float x, y, z;
+
+        for (size_t i  = 0; i < pointsGood_.size(); ++i)
+        {
+            if (mask[i])
+            {
+                p0 = pointsPrevGood_[i]; p1 = pointsGood_[i];
+                x = M(0,0)*p0.x + M(0,1)*p0.y + M(0,2);
+                y = M(1,0)*p0.x + M(1,1)*p0.y + M(1,2);
+                z = M(2,0)*p0.x + M(2,1)*p0.y + M(2,2);
+                x /= z; y /= z;
+                rmse += sqr(x - p1.x) + sqr(y - p1.y);
+                ninliers++;
+            }
+        }
+
+        rmse = sqrt(rmse / static_cast<float>(ninliers));
+    }
 
     if (rmse > maxRmse_ || static_cast<float>(ninliers) / pointsGood_.size() < minInlierRatio_)
         M = Mat::eye(3, 3, CV_32F);

modules/videostab/src/inpainting.cpp

@@ -70,6 +70,14 @@ void InpaintingPipeline::setFrames(const vector<Mat> &val)
 }
 
 
+void InpaintingPipeline::setMotionModel(MotionModel val)
+{
+    for (size_t i = 0; i < inpainters_.size(); ++i)
+        inpainters_[i]->setMotionModel(val);
+    InpainterBase::setMotionModel(val);
+}
+
+
 void InpaintingPipeline::setMotions(const vector<Mat> &val)
 {
     for (size_t i = 0; i < inpainters_.size(); ++i)
@@ -361,17 +369,34 @@ void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask)
 
         Mat motion1to0 = motions[radius_ + neighbor - idx].inv();
 
-        frame1_ = at(neighbor, *frames_);
-        warpAffine(
-                frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(),
-                INTER_LINEAR, borderMode_);
+        // warp frame
+
+        frame1_ = at(neighbor, *frames_);        
+
+        if (motionModel_ != HOMOGRAPHY)
+            warpAffine(
+                    frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(),
+                    INTER_LINEAR, borderMode_);
+        else
+            warpPerspective(
+                    frame1_, transformedFrame1_, motion1to0, frame1_.size(), INTER_LINEAR,
+                    borderMode_);
+
         cvtColor(transformedFrame1_, transformedGrayFrame1_, CV_BGR2GRAY);
 
-        warpAffine(
-                mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(),
-                INTER_NEAREST);
+        // warp mask
+
+        if (motionModel_ != HOMOGRAPHY)
+            warpAffine(
+                    mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(),
+                    INTER_NEAREST);
+        else
+            warpPerspective(mask1_, transformedMask1_, motion1to0, mask1_.size(), INTER_NEAREST);
+
         erode(transformedMask1_, transformedMask1_, Mat());
 
+        // update flow
+
         optFlowEstimator_->run(grayFrame_, transformedGrayFrame1_, flowX_, flowY_, flowErrors_);
 
         calcFlowMask(

modules/videostab/src/precomp.hpp

@@ -54,6 +54,7 @@
 #include "opencv2/imgproc/imgproc.hpp"
 #include "opencv2/video/video.hpp"
 #include "opencv2/features2d/features2d.hpp"
+#include "opencv2/calib3d/calib3d.hpp"
 
 // some aux. functions
 

modules/videostab/src/stabilizer.cpp

@@ -71,6 +71,7 @@ void StabilizerBase::setUp(int cacheSize, const Mat &frame)
     doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
     if (doInpainting_)
     {
+        inpainter_->setMotionModel(motionEstimator_->motionModel());
         inpainter_->setFrames(frames_);
         inpainter_->setMotions(motions_);
         inpainter_->setStabilizedFrames(stabilizedFrames_);
@@ -176,15 +177,26 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
         preProcessedFrame_ = at(curStabilizedPos_, frames_);
 
     // apply stabilization transformation
-    warpAffine(
-            preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
-            stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
+
+    if (motionEstimator_->motionModel() != HOMOGRAPHY)
+        warpAffine(
+                preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
+                stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
+    else
+        warpPerspective(
+                preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
+                stabilizationMotion_, frameSize_, INTER_LINEAR, borderMode_);
 
     if (doInpainting_)
     {
-        warpAffine(
-                frameMask_, at(curStabilizedPos_, stabilizedMasks_),
-                stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
+        if (motionEstimator_->motionModel() != HOMOGRAPHY)
+            warpAffine(
+                    frameMask_, at(curStabilizedPos_, stabilizedMasks_),
+                    stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
+        else
+            warpPerspective(
+                    frameMask_, at(curStabilizedPos_, stabilizedMasks_),
+                    stabilizationMotion_, frameSize_, INTER_NEAREST);
 
         erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_),
               Mat());

samples/cpp/videostab.cpp

@@ -29,7 +29,7 @@ void run();
 void saveMotionsIfNecessary();
 void printHelp();
 
-class GlobalMotionReader : public IGlobalMotionEstimator
+class GlobalMotionReader : public GlobalMotionEstimatorBase
 {
 public:
     GlobalMotionReader(string path)
@@ -256,11 +256,11 @@ int main(int argc, const char **argv)
         {
             RansacParams ransac;
             PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator();
-            Ptr<IGlobalMotionEstimator> est_(est);
+            Ptr<GlobalMotionEstimatorBase> est_(est);
             if (arg("model") == "transl")
             {
                 est->setMotionModel(TRANSLATION);
-                ransac = RansacParams::translationMotionStd();
+                ransac = RansacParams::translation2dMotionStd();
             }
             else if (arg("model") == "transl_and_scale")
             {
@@ -270,13 +270,13 @@ int main(int argc, const char **argv)
             else if (arg("model") == "linear_sim")
             {
                 est->setMotionModel(LINEAR_SIMILARITY);
-                ransac = RansacParams::linearSimilarityMotionStd();
+                ransac = RansacParams::linearSimilarity2dMotionStd();
             }
             else if (arg("model") == "affine")
             {
                 est->setMotionModel(AFFINE);
                 ransac = RansacParams::affine2dMotionStd();
-            }
+            }            
             else
                 throw runtime_error("unknown motion model: " + arg("model"));
             ransac.eps = argf("outlier-ratio");