Refactored videostab module. Added normalization into motion estimators.

modules/videostab/include/opencv2/videostab/global_motion.hpp

@@ -66,8 +66,7 @@ enum MotionModel
 };
 
 CV_EXPORTS Mat estimateGlobalMotionLeastSquares(
-        const std::vector<Point2f> &points0, const std::vector<Point2f> &points1,
-        int model = AFFINE, float *rmse = 0);
+        int npoints, Point2f *points0, Point2f *points1, int model = AFFINE, float *rmse = 0);
 
 struct CV_EXPORTS RansacParams
 {
@@ -80,16 +79,24 @@ struct CV_EXPORTS RansacParams
     RansacParams(int size, float thresh, float eps, float prob)
         : size(size), thresh(thresh), eps(eps), prob(prob) {}
 
-    static RansacParams translation2dMotionStd() { return RansacParams(1, 0.5f, 0.5f, 0.99f); }
-    static RansacParams translationAndScale2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); }
-    static RansacParams linearSimilarity2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); }
-    static RansacParams affine2dMotionStd() { return RansacParams(3, 0.5f, 0.5f, 0.99f); }
-    static RansacParams homography2dMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); }
+    static RansacParams default2dMotion(MotionModel model)
+    {
+        CV_Assert(model < UNKNOWN);
+        if (model == TRANSLATION)
+            return RansacParams(1, 0.5f, 0.5f, 0.99f);
+        if (model == TRANSLATION_AND_SCALE)
+            return RansacParams(2, 0.5f, 0.5f, 0.99f);
+        if (model == LINEAR_SIMILARITY)
+            return RansacParams(2, 0.5f, 0.5f, 0.99f);
+        if (model == AFFINE)
+            return RansacParams(3, 0.5f, 0.5f, 0.99f);
+        return RansacParams(4, 0.5f, 0.5f, 0.99f);
+    }
 };
 
 CV_EXPORTS Mat estimateGlobalMotionRobust(
         const std::vector<Point2f> &points0, const std::vector<Point2f> &points1,
-        int model = AFFINE, const RansacParams &params = RansacParams::affine2dMotionStd(),
+        int model = AFFINE, const RansacParams &params = RansacParams::default2dMotion(AFFINE),
         float *rmse = 0, int *ninliers = 0);
 
 class CV_EXPORTS GlobalMotionEstimatorBase

modules/videostab/src/global_motion.cpp

@@ -51,8 +51,44 @@ namespace cv
 namespace videostab
 {
 
+// does isotropic normalization
+static Mat normalizePoints(int npoints, Point2f *points)
+{
+    float cx = 0.f, cy = 0.f;
+    for (int i = 0; i < npoints; ++i)
+    {
+        cx += points[i].x;
+        cy += points[i].y;
+    }
+    cx /= npoints;
+    cy /= npoints;
+
+    float d = 0.f;
+    for (int i = 0; i < npoints; ++i)
+    {
+        points[i].x -= cx;
+        points[i].y -= cy;
+        d += sqrt(sqr(points[i].x) + sqr(points[i].y));
+    }
+    d /= npoints;
+
+    float s = sqrt(2.f) / d;
+    for (int i = 0; i < npoints; ++i)
+    {
+        points[i].x *= s;
+        points[i].y *= s;
+    }
+
+    Mat_<float> T = Mat::eye(3, 3, CV_32F);
+    T(0,0) = T(1,1) = s;
+    T(0,2) = -cx*s;
+    T(1,2) = -cy*s;
+    return T;
+}
+
+
 static Mat estimateGlobMotionLeastSquaresTranslation(
-        int npoints, const Point2f *points0, const Point2f *points1, float *rmse)
+        int npoints, Point2f *points0, Point2f *points1, float *rmse)
 {
     Mat_<float> M = Mat::eye(3, 3, CV_32F);
     for (int i = 0; i < npoints; ++i)
@@ -62,6 +98,7 @@ static Mat estimateGlobMotionLeastSquaresTranslation(
     }
     M(0,2) /= npoints;
     M(1,2) /= npoints;
+
     if (rmse)
     {
         *rmse = 0;
@@ -70,13 +107,17 @@ static Mat estimateGlobMotionLeastSquaresTranslation(
                      sqr(points1[i].y - points0[i].y - M(1,2));
         *rmse = sqrt(*rmse / npoints);
     }
+
     return M;
 }
 
 
 static Mat estimateGlobMotionLeastSquaresTranslationAndScale(
-        int npoints, const Point2f *points0, const Point2f *points1, float *rmse)
+        int npoints, Point2f *points0, Point2f *points1, float *rmse)
 {
+    Mat_<float> T0 = normalizePoints(npoints, points0);
+    Mat_<float> T1 = normalizePoints(npoints, points1);
+
     Mat_<float> A(2*npoints, 3), b(2*npoints, 1);
     float *a0, *a1;
     Point2f p0, p1;
@@ -103,13 +144,17 @@ static Mat estimateGlobMotionLeastSquaresTranslationAndScale(
     M(0,0) = M(1,1) = sol(0,0);
     M(0,2) = sol(1,0);
     M(1,2) = sol(2,0);
-    return M;
+
+    return T1.inv() * M * T0;
 }
 
 
 static Mat estimateGlobMotionLeastSquaresLinearSimilarity(
-        int npoints, const Point2f *points0, const Point2f *points1, float *rmse)
+        int npoints, Point2f *points0, Point2f *points1, float *rmse)
 {
+    Mat_<float> T0 = normalizePoints(npoints, points0);
+    Mat_<float> T1 = normalizePoints(npoints, points1);
+
     Mat_<float> A(2*npoints, 4), b(2*npoints, 1);
     float *a0, *a1;
     Point2f p0, p1;
@@ -138,13 +183,17 @@ static Mat estimateGlobMotionLeastSquaresLinearSimilarity(
     M(1,0) = -sol(1,0);
     M(0,2) = sol(2,0);
     M(1,2) = sol(3,0);
-    return M;
+
+    return T1.inv() * M * T0;
 }
 
 
 static Mat estimateGlobMotionLeastSquaresAffine(
-        int npoints, const Point2f *points0, const Point2f *points1, float *rmse)
+        int npoints, Point2f *points0, Point2f *points1, float *rmse)
 {
+    Mat_<float> T0 = normalizePoints(npoints, points0);
+    Mat_<float> T1 = normalizePoints(npoints, points1);
+
     Mat_<float> A(2*npoints, 6), b(2*npoints, 1);
     float *a0, *a1;
     Point2f p0, p1;
@@ -172,24 +221,22 @@ static Mat estimateGlobMotionLeastSquaresAffine(
         for (int j = 0; j < 3; ++j, ++k)
             M(i,j) = sol(k,0);
 
-    return M;
+    return T1.inv() * M * T0;
 }
 
 
 Mat estimateGlobalMotionLeastSquares(
-        const vector<Point2f> &points0, const vector<Point2f> &points1, int model, float *rmse)
+        int npoints, Point2f *points0, Point2f *points1, int model, float *rmse)
 {
     CV_Assert(model <= AFFINE);
-    CV_Assert(points0.size() == points1.size());
 
-    typedef Mat (*Impl)(int, const Point2f*, const Point2f*, float*);
+    typedef Mat (*Impl)(int, Point2f*, Point2f*, float*);
     static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation,
                             estimateGlobMotionLeastSquaresTranslationAndScale,
                             estimateGlobMotionLeastSquaresLinearSimilarity,
                             estimateGlobMotionLeastSquaresAffine };
 
-    int npoints = static_cast<int>(points0.size());
-    return impls[model](npoints, &points0[0], &points1[0], rmse);
+    return impls[model](npoints, points0, points1, rmse);
 }
 
 
@@ -200,22 +247,22 @@ Mat estimateGlobalMotionRobust(
     CV_Assert(model <= AFFINE);
     CV_Assert(points0.size() == points1.size());
 
-    typedef Mat (*Impl)(int, const Point2f*, const Point2f*, float*);
-    static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation,
-                            estimateGlobMotionLeastSquaresTranslationAndScale,
-                            estimateGlobMotionLeastSquaresLinearSimilarity,
-                            estimateGlobMotionLeastSquaresAffine };
-
     const int npoints = static_cast<int>(points0.size());
     const int niters = static_cast<int>(ceil(log(1 - params.prob) /
                                              log(1 - pow(1 - params.eps, params.size))));
 
-    RNG rng(0);
+    // current hypothesis
     vector<int> indices(params.size);
-    vector<Point2f> subset0(params.size), subset1(params.size);
-    vector<Point2f> subset0best(params.size), subset1best(params.size);
+    vector<Point2f> subset0(params.size);
+    vector<Point2f> subset1(params.size);
+
+    // best hypothesis
+    vector<Point2f> subset0best(params.size);
+    vector<Point2f> subset1best(params.size);
     Mat_<float> bestM;
     int ninliersMax = -1;
+
+    RNG rng(0);
     Point2f p0, p1;
     float x, y;
 
@@ -239,7 +286,8 @@ Mat estimateGlobalMotionRobust(
             subset1[i] = points1[indices[i]];
         }
 
-        Mat_<float> M = impls[model](params.size, &subset0[0], &subset1[0], 0);
+        Mat_<float> M = estimateGlobalMotionLeastSquares(
+                params.size, &subset0[0], &subset1[0], model, 0);
 
         int ninliers = 0;
         for (int i = 0; i < npoints; ++i)
@@ -260,8 +308,9 @@ Mat estimateGlobalMotionRobust(
     }
 
     if (ninliersMax < params.size)
-        // compute rmse
-        bestM = impls[model](params.size, &subset0best[0], &subset1best[0], rmse);
+        // compute RMSE
+        bestM = estimateGlobalMotionLeastSquares(
+                params.size, &subset0best[0], &subset1best[0], model, rmse);
     else
     {
         subset0.resize(ninliersMax);
@@ -278,7 +327,8 @@ Mat estimateGlobalMotionRobust(
                 j++;
             }
         }
-        bestM = impls[model](ninliersMax, &subset0[0], &subset1[0], rmse);
+        bestM = estimateGlobalMotionLeastSquares(
+                ninliersMax, &subset0[0], &subset1[0], model, rmse);
     }
 
     if (ninliers)
@@ -332,16 +382,11 @@ PyrLkRobustMotionEstimator::PyrLkRobustMotionEstimator(MotionModel model)
     setDetector(new GoodFeaturesToTrackDetector());
     setOptFlowEstimator(new SparsePyrLkOptFlowEstimator());
     setMotionModel(model);
-    if (model == TRANSLATION)
-        setRansacParams(RansacParams::translation2dMotionStd());
-    else if (model == TRANSLATION_AND_SCALE)
-        setRansacParams(RansacParams::translationAndScale2dMotionStd());
-    else if (model == LINEAR_SIMILARITY)
-        setRansacParams(RansacParams::linearSimilarity2dMotionStd());
-    else if (model == AFFINE)
-        setRansacParams(RansacParams::affine2dMotionStd());
-    else if (model == HOMOGRAPHY)
-        setRansacParams(RansacParams::homography2dMotionStd());
+
+    RansacParams ransac = RansacParams::default2dMotion(model);
+    ransac.size *= 2; // we use more points than needed, but result looks better
+    setRansacParams(ransac);
+
     setMaxRmse(0.5f);
     setMinInlierRatio(0.1f);
     setGridSize(Size(0,0));
@@ -362,6 +407,7 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b
                 keypointsPrev_.push_back(KeyPoint((x+1)*dx, (y+1)*dy, 0.f));
     }
 
+    // draw keypoints
     /*Mat img;
     drawKeypoints(frame0, keypointsPrev_, img);
     imshow("frame0_keypoints", img);
@@ -374,10 +420,9 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b
     optFlowEstimator_->run(frame0, frame1, pointsPrev_, points_, status_, noArray());
 
     size_t npoints = points_.size();
-    pointsPrevGood_.clear();
-    pointsPrevGood_.reserve(npoints);
-    pointsGood_.clear();
-    pointsGood_.reserve(npoints);
+    pointsPrevGood_.clear(); pointsPrevGood_.reserve(npoints);
+    pointsGood_.clear(); pointsGood_.reserve(npoints);
+
     for (size_t i = 0; i < npoints; ++i)
     {
         if (status_[i])

modules/videostab/src/stabilizer.cpp

@@ -338,16 +338,18 @@ void TwoPassStabilizer::runPrePassIfNecessary()
         WobbleSuppressorBase *wobbleSuppressor = static_cast<WobbleSuppressorBase*>(wobbleSuppressor_);
         doWobbleSuppression_ = dynamic_cast<NullWobbleSuppressor*>(wobbleSuppressor) == 0;
 
-        bool okEst;
+        bool ok = true, ok2 = true;
+
         while (!(frame = frameSource_->nextFrame()).empty())
         {
             if (frameCount_ > 0)
             {
-                motions_.push_back(motionEstimator_->estimate(prevFrame, frame));
+                motions_.push_back(motionEstimator_->estimate(prevFrame, frame, &ok));
+
                 if (doWobbleSuppression_)
                 {
-                    Mat M = wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame, &okEst);
-                    if (okEst)
+                    Mat M = wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame, &ok2);
+                    if (ok2)
                         motions2_.push_back(M);
                     else
                         motions2_.push_back(motions_.back());
@@ -363,7 +365,12 @@ void TwoPassStabilizer::runPrePassIfNecessary()
             prevFrame = frame;
             frameCount_++;
 
-            log_->print(".");
+            if (ok)
+            {
+                if (ok2) log_->print(".");
+                else log_->print("?");
+            }
+            else log_->print("x");
         }
 
         for (int i = 0; i < radius_; ++i)

modules/videostab/src/wobble_suppression.cpp

@@ -55,7 +55,7 @@ WobbleSuppressorBase::WobbleSuppressorBase() : motions_(0), stabilizationMotions
 {
     PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator();
     est->setMotionModel(HOMOGRAPHY);
-    est->setRansacParams(RansacParams::homography2dMotionStd());
+    est->setRansacParams(RansacParams::default2dMotion(HOMOGRAPHY));
     setMotionEstimator(est);
 }
 

samples/cpp/videostab.cpp

@@ -67,6 +67,8 @@ void printHelp()
             "Arguments:\n"
             "  -m, --model=(transl|transl_and_scale|linear_sim|affine|homography)\n"
             "      Set motion model. The default is affine.\n"
+            "  --subset=(<int_number>|auto)\n"
+            "      Number of random samples per one motion hypothesis. The default is auto.\n"
             "  --outlier-ratio=<float_number>\n"
             "      Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
             "  --min-inlier-ratio=<float_number>\n"
@@ -116,10 +118,12 @@ void printHelp()
             "  --ws-model=(transl|transl_and_scale|linear_sim|affine|homography)\n"
             "      Set wobble suppression motion model (must have more DOF than motion \n"
             "      estimation model). The default is homography.\n"
-            "  --ws-min-inlier-ratio=<float_number>\n"
-            "      Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
+            "  --ws-subset=(<int_number>|auto)\n"
+            "      Number of random samples per one motion hypothesis. The default is auto.\n"
             "  --ws-outlier-ratio=<float_number>\n"
             "      Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
+            "  --ws-min-inlier-ratio=<float_number>\n"
+            "      Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
             "  --ws-nkps=<int_number>\n"
             "      Number of keypoints to find in each frame. The default is 1000.\n"
             "  --ws-extra-kps=<int_number>\n"
@@ -147,8 +151,9 @@ int main(int argc, const char **argv)
         const char *keys =
                 "{ 1 | | | | }"
                 "{ m | model | affine| }"
-                "{ | min-inlier-ratio | 0.1 | }"
+                "{ | subset | auto | }"
                 "{ | outlier-ratio | 0.5 | }"
+                "{ | min-inlier-ratio | 0.1 | }"
                 "{ | nkps | 1000 | }"
                 "{ | extra-kps | 0 | }"
                 "{ sm | save-motions | no | }"
@@ -170,8 +175,9 @@ int main(int argc, const char **argv)
                 "{ ws | wobble-suppress | no | }"
                 "{ | ws-period | 30 | }"
                 "{ | ws-model | homography | }"
-                "{ | ws-min-inlier-ratio | 0.1 | }"
+                "{ | ws-subset | auto | }"
                 "{ | ws-outlier-ratio | 0.5 | }"
+                "{ | ws-min-inlier-ratio | 0.1 | }"
                 "{ | ws-nkps | 1000 | }"
                 "{ | ws-extra-kps | 0 | }"
                 "{ sm2 | save-motions2 | no | }"
@@ -230,6 +236,8 @@ int main(int argc, const char **argv)
 
                 est->setDetector(new GoodFeaturesToTrackDetector(argi("ws-nkps")));
                 RansacParams ransac = est->ransacParams();
+                if (arg("ws-subset") != "auto")
+                    ransac.size = argi("ws-subset");
                 ransac.eps = argf("ws-outlier-ratio");
                 est->setRansacParams(ransac);
                 est->setMinInlierRatio(argf("ws-min-inlier-ratio"));
@@ -288,6 +296,8 @@ int main(int argc, const char **argv)
 
         est->setDetector(new GoodFeaturesToTrackDetector(argi("nkps")));
         RansacParams ransac = est->ransacParams();
+        if (arg("subset") != "auto")
+            ransac.size = argi("subset");
         ransac.eps = argf("outlier-ratio");
         est->setRansacParams(ransac);
         est->setMinInlierRatio(argf("min-inlier-ratio"));