Added linear programming based stabilizer (videostab)

modules/videostab/include/opencv2/videostab/global_motion.hpp

@@ -57,16 +57,16 @@ namespace videostab
 
 enum MotionModel
 {
-    TRANSLATION = 0,
-    TRANSLATION_AND_SCALE = 1,
-    LINEAR_SIMILARITY = 2,
-    AFFINE = 3,
-    HOMOGRAPHY = 4,
-    UNKNOWN = 5
+    MM_TRANSLATION = 0,
+    MM_TRANSLATION_AND_SCALE = 1,
+    MM_LINEAR_SIMILARITY = 2,
+    MM_AFFINE = 3,
+    MM_HOMOGRAPHY = 4,
+    MM_UNKNOWN = 5
 };
 
 CV_EXPORTS Mat estimateGlobalMotionLeastSquares(
-        int npoints, Point2f *points0, Point2f *points1, int model = AFFINE, float *rmse = 0);
+        int npoints, Point2f *points0, Point2f *points1, int model = MM_AFFINE, float *rmse = 0);
 
 struct CV_EXPORTS RansacParams
 {
@@ -81,14 +81,14 @@ struct CV_EXPORTS RansacParams
 
     static RansacParams default2dMotion(MotionModel model)
     {
-        CV_Assert(model < UNKNOWN);
-        if (model == TRANSLATION)
+        CV_Assert(model < MM_UNKNOWN);
+        if (model == MM_TRANSLATION)
             return RansacParams(1, 0.5f, 0.5f, 0.99f);
-        if (model == TRANSLATION_AND_SCALE)
+        if (model == MM_TRANSLATION_AND_SCALE)
             return RansacParams(2, 0.5f, 0.5f, 0.99f);
-        if (model == LINEAR_SIMILARITY)
+        if (model == MM_LINEAR_SIMILARITY)
             return RansacParams(2, 0.5f, 0.5f, 0.99f);
-        if (model == AFFINE)
+        if (model == MM_AFFINE)
             return RansacParams(3, 0.5f, 0.5f, 0.99f);
         return RansacParams(4, 0.5f, 0.5f, 0.99f);
     }
@@ -96,13 +96,13 @@ struct CV_EXPORTS RansacParams
 
 CV_EXPORTS Mat estimateGlobalMotionRobust(
         const std::vector<Point2f> &points0, const std::vector<Point2f> &points1,
-        int model = AFFINE, const RansacParams &params = RansacParams::default2dMotion(AFFINE),
+        int model = MM_AFFINE, const RansacParams &params = RansacParams::default2dMotion(MM_AFFINE),
         float *rmse = 0, int *ninliers = 0);
 
 class CV_EXPORTS GlobalMotionEstimatorBase
 {
 public:
-    GlobalMotionEstimatorBase() : motionModel_(UNKNOWN) {}
+    GlobalMotionEstimatorBase() : motionModel_(MM_UNKNOWN) {}
     virtual ~GlobalMotionEstimatorBase() {}
 
     virtual void setMotionModel(MotionModel val) { motionModel_ = val; }
@@ -138,7 +138,7 @@ private:
 class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase
 {
 public:
-    PyrLkRobustMotionEstimator(MotionModel model = AFFINE);
+    PyrLkRobustMotionEstimator(MotionModel model = MM_AFFINE);
 
     void setDetector(Ptr<FeatureDetector> val) { detector_ = val; }
     Ptr<FeatureDetector> detector() const { return detector_; }

modules/videostab/include/opencv2/videostab/inpainting.hpp

@@ -59,7 +59,7 @@ class CV_EXPORTS InpainterBase
 {
 public:
     InpainterBase()
-        : radius_(0), motionModel_(UNKNOWN), frames_(0), motions_(0),
+        : radius_(0), motionModel_(MM_UNKNOWN), frames_(0), motions_(0),
           stabilizedFrames_(0), stabilizationMotions_(0) {}
 
     virtual ~InpainterBase() {}

modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp

@@ -61,7 +61,7 @@ public:
     // assumes that [0, size-1) is in or equals to [range.first, range.second)
     virtual void stabilize(
             int size, const std::vector<Mat> &motions, std::pair<int,int> range,
-            Mat *stabilizationMotions) const = 0;
+            Mat *stabilizationMotions) = 0;
 };
 
 class CV_EXPORTS MotionStabilizationPipeline : public IMotionStabilizer
@@ -72,7 +72,7 @@ public:
 
     virtual void stabilize(
             int size, const std::vector<Mat> &motions, std::pair<int,int> range,
-            Mat *stabilizationMotions) const;
+            Mat *stabilizationMotions);
 
 private:
     std::vector<Ptr<IMotionStabilizer> > stabilizers_;
@@ -84,11 +84,11 @@ public:
     virtual ~MotionFilterBase() {}
 
     virtual Mat stabilize(
-            int idx, const std::vector<Mat> &motions, std::pair<int,int> range) const = 0;
+            int idx, const std::vector<Mat> &motions, std::pair<int,int> range) = 0;
 
     virtual void stabilize(
             int size, const std::vector<Mat> &motions, std::pair<int,int> range,
-            Mat *stabilizationMotions) const;
+            Mat *stabilizationMotions);
 };
 
 class CV_EXPORTS GaussianMotionFilter : public MotionFilterBase
@@ -101,7 +101,7 @@ public:
     float stdev() const { return stdev_; }
 
     virtual Mat stabilize(
-            int idx, const std::vector<Mat> &motions, std::pair<int,int> range) const;
+            int idx, const std::vector<Mat> &motions, std::pair<int,int> range);
 
 private:
     int radius_;
@@ -112,17 +112,49 @@ private:
 class CV_EXPORTS LpMotionStabilizer : public IMotionStabilizer
 {
 public:
-    LpMotionStabilizer(MotionModel model = LINEAR_SIMILARITY);
+    LpMotionStabilizer(MotionModel model = MM_LINEAR_SIMILARITY);
 
     void setMotionModel(MotionModel val) { model_ = val; }
     MotionModel motionModel() const { return model_; }
 
+    void setFrameSize(Size val) { frameSize_ = val; }
+    Size frameSize() const { return frameSize_; }
+
+    void setTrimRatio(float val) { trimRatio_ = val; }
+    float trimRatio() const { return trimRatio_; }
+
+    void setWeight1(float val) { w1_ = val; }
+    float weight1() const { return w1_; }
+
+    void setWeight2(float val) { w2_ = val; }
+    float weight2() const { return w2_; }
+
+    void setWeight3(float val) { w3_ = val; }
+    float weight3() const { return w3_; }
+
+    void setWeight4(float val) { w4_ = val; }
+    float weight4() const { return w4_; }
+
     virtual void stabilize(
             int size, const std::vector<Mat> &motions, std::pair<int,int> range,
-            Mat *stabilizationMotions) const;
+            Mat *stabilizationMotions);
 
 private:
     MotionModel model_;
+    Size frameSize_;
+    float trimRatio_;
+    float w1_, w2_, w3_, w4_;
+
+    std::vector<double> obj_, collb_, colub_;
+    std::vector<int> rows_, cols_;
+    std::vector<double> elems_, rowlb_, rowub_;
+
+    void set(int row, int col, double coef)
+    {
+        rows_.push_back(row);
+        cols_.push_back(col);
+        elems_.push_back(coef);
+    }
 };
 
 CV_EXPORTS Mat ensureInclusionConstraint(const Mat &M, Size size, float trimRatio);

modules/videostab/src/global_motion.cpp

@@ -228,7 +228,7 @@ static Mat estimateGlobMotionLeastSquaresAffine(
 Mat estimateGlobalMotionLeastSquares(
         int npoints, Point2f *points0, Point2f *points1, int model, float *rmse)
 {
-    CV_Assert(model <= AFFINE);
+    CV_Assert(model <= MM_AFFINE);
 
     typedef Mat (*Impl)(int, Point2f*, Point2f*, float*);
     static Impl impls[] = { estimateGlobMotionLeastSquaresTranslation,
@@ -244,7 +244,7 @@ Mat estimateGlobalMotionRobust(
         const vector<Point2f> &points0, const vector<Point2f> &points1, int model,
         const RansacParams &params, float *rmse, int *ninliers)
 {
-    CV_Assert(model <= AFFINE);
+    CV_Assert(model <= MM_AFFINE);
     CV_Assert(points0.size() == points1.size());
 
     const int npoints = static_cast<int>(points0.size());
@@ -436,7 +436,7 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1, b
     int ninliers;
     Mat_<float> M;
 
-    if (motionModel_ != HOMOGRAPHY)
+    if (motionModel_ != MM_HOMOGRAPHY)
         M = estimateGlobalMotionRobust(
                 pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, &rmse, &ninliers);
     else

modules/videostab/src/inpainting.cpp

@@ -375,7 +375,7 @@ void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask)
 
         frame1_ = at(neighbor, *frames_);        
 
-        if (motionModel_ != HOMOGRAPHY)
+        if (motionModel_ != MM_HOMOGRAPHY)
             warpAffine(
                     frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(),
                     INTER_LINEAR, borderMode_);
@@ -388,7 +388,7 @@ void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask)
 
         // warp mask
 
-        if (motionModel_ != HOMOGRAPHY)
+        if (motionModel_ != MM_HOMOGRAPHY)
             warpAffine(
                     mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(),
                     INTER_NEAREST);

modules/videostab/src/stabilizer.cpp

@@ -194,7 +194,7 @@ void StabilizerBase::stabilizeFrame()
 
     // apply stabilization transformation
 
-    if (motionEstimator_->motionModel() != HOMOGRAPHY)
+    if (motionEstimator_->motionModel() != MM_HOMOGRAPHY)
         warpAffine(
                 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
                 stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
@@ -205,7 +205,7 @@ void StabilizerBase::stabilizeFrame()
 
     if (doInpainting_)
     {
-        if (motionEstimator_->motionModel() != HOMOGRAPHY)
+        if (motionEstimator_->motionModel() != MM_HOMOGRAPHY)
             warpAffine(
                     frameMask_, at(curStabilizedPos_, stabilizedMasks_),
                     stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);

modules/videostab/src/wobble_suppression.cpp

@@ -54,8 +54,8 @@ namespace videostab
 WobbleSuppressorBase::WobbleSuppressorBase() : motions_(0), stabilizationMotions_(0)
 {
     PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator();
-    est->setMotionModel(HOMOGRAPHY);
-    est->setRansacParams(RansacParams::default2dMotion(HOMOGRAPHY));
+    est->setMotionModel(MM_HOMOGRAPHY);
+    est->setRansacParams(RansacParams::default2dMotion(MM_HOMOGRAPHY));
     setMotionEstimator(est);
 }
 

samples/cpp/videostab.cpp

@@ -86,6 +86,18 @@ void printHelp()
             "  --stdev=(<float_number>|auto)\n"
             "      Set smoothing weights standard deviation. The default is auto\n"
             "      (i.e. sqrt(radius)).\n"
+            "  -lp, --lp-stab=(yes|no)\n"
+            "      Turn on/off linear programming based stabilization method.\n"
+            "  --lp-trim-ratio=(<float_number>|auto)\n"
+            "      Trimming ratio used in linear programming based method.\n"
+            "  --lp-w1=(<float_number>|1)\n"
+            "      1st derivative weight. The default is 1.\n"
+            "  --lp-w2=(<float_number>|10)\n"
+            "      2nd derivative weight. The default is 10.\n"
+            "  --lp-w3=(<float_number>|100)\n"
+            "      3rd derivative weight. The default is 100.\n"
+            "  --lp-w4=(<float_number>|100)\n"
+            "      Non-translation motion components weight. The default is 100.\n\n"
             "  --deblur=(yes|no)\n"
             "      Do deblurring.\n"
             "  --deblur-sens=<float_number>\n"
@@ -160,6 +172,12 @@ int main(int argc, const char **argv)
                 "{ lm | load-motions | no | }"
                 "{ r | radius | 15 | }"
                 "{ | stdev | auto | }"
+                "{ lp | lp-stab | no | }"
+                "{ | lp-trim-ratio | auto | }"
+                "{ | lp-w1 | 1 | }"
+                "{ | lp-w2 | 10 | }"
+                "{ | lp-w3 | 100 | }"
+                "{ | lp-w4 | 100 | }"
                 "{ | deblur | no | }"
                 "{ | deblur-sens | 0.1 | }"
                 "{ et | est-trim | yes | }"
@@ -194,19 +212,37 @@ int main(int argc, const char **argv)
         {
             printHelp();
             return 0;
-        }               
+        }
+
+        string inputPath = arg("1");
+        if (inputPath.empty()) throw runtime_error("specify video file path");
+
+        VideoFileSource *source = new VideoFileSource(inputPath);
+        cout << "frame count (rough): " << source->count() << endl;
+        if (arg("fps") == "auto") outputFps = source->fps();  else outputFps = argd("fps");
 
         StabilizerBase *stabilizer;
 
         bool isTwoPass =
-                arg("est-trim") == "yes" || arg("wobble-suppress") == "yes";
+                arg("est-trim") == "yes" || arg("wobble-suppress") == "yes" || arg("lp-stab") == "yes";
 
         if (isTwoPass)
         {
             TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer();
             stabilizer = twoPassStabilizer;
             twoPassStabilizer->setEstimateTrimRatio(arg("est-trim") == "yes");
-            if (arg("stdev") == "auto")
+            if (arg("lp-stab") == "yes")
+            {
+                LpMotionStabilizer *stab = new LpMotionStabilizer();
+                stab->setFrameSize(Size(source->width(), source->height()));
+                stab->setTrimRatio(arg("lp-trim-ratio") == "auto" ? argf("trim-ratio") : argf("lp-trim-ratio"));
+                stab->setWeight1(argf("lp-w1"));
+                stab->setWeight2(argf("lp-w2"));
+                stab->setWeight3(argf("lp-w3"));
+                stab->setWeight4(argf("lp-w4"));
+                twoPassStabilizer->setMotionStabilizer(stab);
+            }
+            else if (arg("stdev") == "auto")
                 twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius")));
             else
                 twoPassStabilizer->setMotionStabilizer(new GaussianMotionFilter(argi("radius"), argf("stdev")));
@@ -219,15 +255,15 @@ int main(int argc, const char **argv)
                 PyrLkRobustMotionEstimator *est = 0;
 
                 if (arg("ws-model") == "transl")
-                    est = new PyrLkRobustMotionEstimator(TRANSLATION);
+                    est = new PyrLkRobustMotionEstimator(MM_TRANSLATION);
                 else if (arg("ws-model") == "transl_and_scale")
-                    est = new PyrLkRobustMotionEstimator(TRANSLATION_AND_SCALE);
+                    est = new PyrLkRobustMotionEstimator(MM_TRANSLATION_AND_SCALE);
                 else if (arg("ws-model") == "linear_sim")
-                    est = new PyrLkRobustMotionEstimator(LINEAR_SIMILARITY);
+                    est = new PyrLkRobustMotionEstimator(MM_LINEAR_SIMILARITY);
                 else if (arg("ws-model") == "affine")
-                    est = new PyrLkRobustMotionEstimator(AFFINE);
+                    est = new PyrLkRobustMotionEstimator(MM_AFFINE);
                 else if (arg("ws-model") == "homography")
-                    est = new PyrLkRobustMotionEstimator(HOMOGRAPHY);
+                    est = new PyrLkRobustMotionEstimator(MM_HOMOGRAPHY);
                 else
                 {
                     delete est;
@@ -266,28 +302,22 @@ int main(int argc, const char **argv)
             else
                 onePassStabilizer->setMotionFilter(new GaussianMotionFilter(argi("radius"), argf("stdev")));
         }
-        stabilizedFrames = dynamic_cast<IFrameSource*>(stabilizer);
-
-        string inputPath = arg("1");
-        if (inputPath.empty()) throw runtime_error("specify video file path");
 
-        VideoFileSource *source = new VideoFileSource(inputPath);
-        cout << "frame count (rough): " << source->count() << endl;
-        if (arg("fps") == "auto") outputFps = source->fps();  else outputFps = argd("fps");
         stabilizer->setFrameSource(source);
+        stabilizedFrames = dynamic_cast<IFrameSource*>(stabilizer);
 
         PyrLkRobustMotionEstimator *est = 0;
 
         if (arg("model") == "transl")
-            est = new PyrLkRobustMotionEstimator(TRANSLATION);
+            est = new PyrLkRobustMotionEstimator(MM_TRANSLATION);
         else if (arg("model") == "transl_and_scale")
-            est = new PyrLkRobustMotionEstimator(TRANSLATION_AND_SCALE);
+            est = new PyrLkRobustMotionEstimator(MM_TRANSLATION_AND_SCALE);
         else if (arg("model") == "linear_sim")
-            est = new PyrLkRobustMotionEstimator(LINEAR_SIMILARITY);
+            est = new PyrLkRobustMotionEstimator(MM_LINEAR_SIMILARITY);
         else if (arg("model") == "affine")
-            est = new PyrLkRobustMotionEstimator(AFFINE);
+            est = new PyrLkRobustMotionEstimator(MM_AFFINE);
         else if (arg("model") == "homography")
-            est = new PyrLkRobustMotionEstimator(HOMOGRAPHY);
+            est = new PyrLkRobustMotionEstimator(MM_HOMOGRAPHY);
         else
         {
             delete est;