Post-review fixes

modules/optflow/include/opencv2/optflow.hpp

@@ -193,8 +193,6 @@ CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_Farneback();
 //! Additional interface to the SparseToDenseFlow algorithm - calcOpticalFlowSparseToDense()
 CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SparseToDense();
 
-CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_BlockMatching();
-
 //! @}
 
 } //optflow

modules/optflow/include/opencv2/optflow/pcaflow.hpp

@@ -8,7 +8,7 @@ copy or use the software.
                For Open Source Computer Vision Library
                        (3-clause BSD License)
 
-Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Copyright (C) 2016, OpenCV Foundation, all rights reserved.
 Third party copyrights are property of their respective owners.
 
 Redistribution and use in source and binary forms, with or without modification,
@@ -37,6 +37,19 @@ or tort (including negligence or otherwise) arising in any way out of
 the use of this software, even if advised of the possibility of such damage.
 */
 
+/*
+Implementation of the PCAFlow algorithm from the following paper:
+http://files.is.tue.mpg.de/black/papers/cvpr2015_pcaflow.pdf
+
+@inproceedings{Wulff:CVPR:2015,
+  title = {Efficient Sparse-to-Dense Optical Flow Estimation using a Learned Basis and Layers},
+  author = {Wulff, Jonas and Black, Michael J.},
+  booktitle = { IEEE Conf. on Computer Vision and Pattern Recognition (CVPR) 2015},
+  month = jun,
+  year = {2015}
+}
+*/
+
 #ifndef __OPENCV_OPTFLOW_PCAFLOW_HPP__
 #define __OPENCV_OPTFLOW_PCAFLOW_HPP__
 
@@ -47,10 +60,13 @@ namespace cv
 {
 namespace optflow
 {
-namespace pcaflow
-{
 
-class Prior
+/*
+ * This class can be used for imposing a learned prior on the resulting optical flow.
+ * Solution will be regularized according to this prior.
+ * You need to generate appropriate prior file with "learn_prior.py" script beforehand.
+ */
+class PCAPrior
 {
 private:
   Mat L1;
@@ -59,7 +75,7 @@ private:
   Mat c2;
 
 public:
-  Prior( const char *pathToPrior );
+  PCAPrior( const char *pathToPrior );
 
   int getPadding() const { return L1.size().height; }
 
@@ -67,12 +83,11 @@ public:
 
   void fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const;
 };
-}
 
 class OpticalFlowPCAFlow : public DenseOpticalFlow
 {
 protected:
-  const pcaflow::Prior *prior;
+  const PCAPrior *prior;
   const Size basisSize;
   const float sparseRate;              // (0 .. 0.1)
   const float retainedCornersFraction; // [0 .. 1]
@@ -80,7 +95,7 @@ protected:
   const float dampingFactor;
 
 public:
-  OpticalFlowPCAFlow( const pcaflow::Prior *_prior = 0, const Size _basisSize = Size( 18, 14 ),
+  OpticalFlowPCAFlow( const PCAPrior *_prior = 0, const Size _basisSize = Size( 18, 14 ),
                       float _sparseRate = 0.02, float _retainedCornersFraction = 0.7,
                       float _occlusionsThreshold = 0.0003, float _dampingFactor = 0.00002 );
 

modules/optflow/samples/optical_flow_evaluation.cpp

@@ -12,7 +12,7 @@ using namespace optflow;
 const String keys = "{help h usage ? |      | print this message   }"
         "{@image1        |      | image1               }"
         "{@image2        |      | image2               }"
-        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, pcaflow, blockmatching or sparsetodenseflow] }"
+        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, pcaflow or sparsetodenseflow] }"
         "{@groundtruth   |      | path to the .flo file  (optional), Middlebury format }"
         "{m measure      |endpoint| error measure - [endpoint or angular] }"
         "{r region       |all   | region to compute stats about [all, discontinuities, untextured] }"
@@ -258,8 +258,6 @@ int main( int argc, char** argv )
         algorithm = createOptFlow_DeepFlow();
     else if ( method == "sparsetodenseflow" )
         algorithm = createOptFlow_SparseToDense();
-    else if ( method == "blockmatching" )
-        algorithm = createOptFlow_BlockMatching();
     else if ( method == "pcaflow" )
         algorithm = createOptFlow_PCAFlow();
     else

modules/optflow/src/block_matching.cpp

-/*M///////////////////////////////////////////////////////////////////////////////////////
- //
- //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
- //
- //  By downloading, copying, installing or using the software you agree to this license.
- //  If you do not agree to this license, do not download, install,
- //  copy or use the software.
- //
- //
- //                           License Agreement
- //                For Open Source Computer Vision Library
- //
- // Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
- // Copyright (C) 2009, Willow Garage Inc., all rights reserved.
- // Third party copyrights are property of their respective owners.
- //
- // Redistribution and use in source and binary forms, with or without modification,
- // are permitted provided that the following conditions are met:
- //
- //   * Redistribution's of source code must retain the above copyright notice,
- //     this list of conditions and the following disclaimer.
- //
- //   * Redistribution's in binary form must reproduce the above copyright notice,
- //     this list of conditions and the following disclaimer in the documentation
- //     and/or other materials provided with the distribution.
- //
- //   * The name of the copyright holders may not be used to endorse or promote products
- //     derived from this software without specific prior written permission.
- //
- // This software is provided by the copyright holders and contributors "as is" and
- // any express or implied warranties, including, but not limited to, the implied
- // warranties of merchantability and fitness for a particular purpose are disclaimed.
- // In no event shall the Intel Corporation or contributors be liable for any direct,
- // indirect, incidental, special, exemplary, or consequential damages
- // (including, but not limited to, procurement of substitute goods or services;
- // loss of use, data, or profits; or business interruption) however caused
- // and on any theory of liability, whether in contract, strict liability,
- // or tort (including negligence or otherwise) arising in any way out of
- // the use of this software, even if advised of the possibility of such damage.
- //
- //M*/
-
-#include "precomp.hpp"
-
-namespace cv
-{
-namespace optflow
-{
-
-class OpticalFlowBlockMatching : public DenseOpticalFlow
-{
-protected:
-  int windowSize;
-  int blockSize;
-
-  inline float submatrixAbsDiff( int x0, int y0, const Mat &I0, int x1, int y1, const Mat &I1 ) const;
-
-public:
-  OpticalFlowBlockMatching() : windowSize( 3 ), blockSize( 8 ){};
-
-  void calc( InputArray I0, InputArray I1, InputOutputArray flow );
-  void collectGarbage();
-};
-
-inline float OpticalFlowBlockMatching::submatrixAbsDiff( int x0, int y0, const Mat &I0, int x1, int y1,
-                                                         const Mat &I1 ) const
-{
-  float error = 0;
-  const Size size = I0.size();
-  for ( int i = -windowSize; i <= windowSize; ++i )
-  {
-    if ( i + y0 < 0 || i + y0 >= size.height || i + y1 < 0 || i + y1 >= size.height )
-    {
-      error += 1;
-      continue;
-    }
-    const Vec3f *I0X = I0.ptr<Vec3f>( i + y0 );
-    const Vec3f *I1X = I1.ptr<Vec3f>( i + y1 );
-    for ( int j = -windowSize; j <= windowSize; ++j )
-    {
-      if ( j + x0 < 0 || j + x0 >= size.width || j + x1 < 0 || j + x1 >= size.width )
-      {
-        error += 1;
-        continue;
-      }
-      const Vec3f diff = I0X[j + x0] - I1X[j + x1];
-      error += abs( diff[0] );
-      error += abs( diff[1] );
-      error += abs( diff[2] );
-    }
-  }
-  return error;
-}
-
-void OpticalFlowBlockMatching::calc( InputArray I0, InputArray I1, InputOutputArray flow_out )
-{
-  CV_Assert( I0.channels() == 3 );
-  CV_Assert( I1.channels() == 3 );
-  Size size = I0.size();
-  CV_Assert( size == I1.size() );
-
-  flow_out.create( size, CV_32FC2 );
-  Mat flow = flow_out.getMat();
-  Mat from = I0.getMat();
-  Mat to = I1.getMat();
-
-  from.convertTo( from, CV_32FC3, 1.0 / 255.0 );
-  to.convertTo( to, CV_32FC3, 1.0 / 255.0 );
-
-  const float distNormalize = blockSize * sqrt( 2 );
-
-  for ( int y0 = 0; y0 < size.height; ++y0 )
-  {
-    Vec2f *flowX = flow.ptr<Vec2f>( y0 );
-    const int yEnd = std::min( size.height - 1, y0 + blockSize );
-
-    for ( int x0 = 0; x0 < size.width; ++x0 )
-    {
-      float minDiff = 1e10;
-      Vec2f du( 0, 0 );
-      const int xEnd = std::min( size.width - 1, x0 + blockSize );
-      for ( int y1 = std::max( 0, y0 - blockSize ); y1 <= yEnd; ++y1 )
-        for ( int x1 = std::max( 0, x0 - blockSize ); x1 <= xEnd; ++x1 )
-        {
-          const float distance = sqrt( ( x0 - x1 ) * ( x0 - x1 ) + ( y0 - y1 ) * ( y0 - y1 ) ) / distNormalize;
-          const float kernel = 1.0 + 0.5 * distance;
-          const float diff = kernel * submatrixAbsDiff( x0, y0, from, x1, y1, to );
-          if ( diff < minDiff )
-          {
-            minDiff = diff;
-            du = Vec2f( ( x1 - x0 ), ( y1 - y0 ) );
-          }
-        }
-      flowX[x0] = du;
-    }
-  }
-}
-
-void OpticalFlowBlockMatching::collectGarbage() {}
-
-Ptr<DenseOpticalFlow> createOptFlow_BlockMatching() { return makePtr<OpticalFlowBlockMatching>(); }
-}
-}

modules/optflow/src/learn_prior.py

-import os, sys
+#!/usr/bin/env python
+
+import os
+import sys
 import numpy as np
 import cv2
 import struct
+import argparse
 from math import sqrt
 
-basis_size = (14, 18)
-lambd = 0.2
+argparser = argparse.ArgumentParser(
+    description='''Use this script to generate prior for using with PCAFlow.
+Basis size here must match corresponding parameter in the PCAFlow.
+Gamma should be selected experimentally.''')
+
+argparser.add_argument('-f',
+                       '--files',
+                       nargs='+',
+                       help='List of optical flow .flo files for learning',
+                       required=True)
+argparser.add_argument('-o',
+                       '--output',
+                       help='Output file for prior',
+                       required=True)
+argparser.add_argument('--width',
+                       type=int,
+                       help='Size of the basis first dimension',
+                       required=True,
+                       default=18)
+argparser.add_argument('--height',
+                       type=int,
+                       help='Size of the basis second dimension',
+                       required=True,
+                       default=14)
+argparser.add_argument(
+    '-g',
+    '--gamma',
+    type=float,
+    help='Amount of regularization. The greater this parameter, the bigger will be an impact of the regularization.',
+    required=True)
+args = argparser.parse_args()
+
+basis_size = (args.height, args.width)
+gamma = args.gamma
+
 
 def find_flo(pp):
     f = []
@@ -14,38 +51,42 @@ def find_flo(pp):
             f.append(p)
         else:
             for root, subdirs, files in os.walk(p):
-				f += map(lambda x: os.path.join(root, x), filter(lambda x: x.split('.')[-1] == 'flo', files))
+                f += map(lambda x: os.path.join(root, x),
+                         filter(lambda x: x.split('.')[-1] == 'flo', files))
     return list(set(f))
 
+
 def load_flo(flo):
     with open(flo) as f:
-            magic = np.fromfile(f, np.float32, count=1)
+        magic = np.fromfile(f, np.float32, count=1)[0]
         if 202021.25 != magic:
             print('Magic number incorrect. Invalid .flo file')
         else:
-                w = np.fromfile(f, np.int32, count=1)
-                h = np.fromfile(f, np.int32, count=1)
+            w = np.fromfile(f, np.int32, count=1)[0]
+            h = np.fromfile(f, np.int32, count=1)[0]
             print('Reading %dx%d flo file %s' % (w, h, flo))
-                data = np.fromfile(f, np.float32, count=2*w*h)
+            data = np.fromfile(f, np.float32, count=2 * w * h)
             # Reshape data into 3D array (columns, rows, bands)
-                flow = np.resize(data, (h, w, 2))
-                return flow[:,:,0], flow[:,:,1]
+            flow = np.reshape(data, (h, w, 2))
+            return flow[:, :, 0], flow[:, :, 1]
+
 
 def get_w(m):
     s = m.shape
     w = cv2.dct(m)
     w *= 2.0 / sqrt(s[0] * s[1])
     #w[0,0] *= 0.5
-	w[:,0] *= sqrt(0.5)
-	w[0,:] *= sqrt(0.5)
-	w = w[0:basis_size[0],0:basis_size[1]].transpose().flatten()
+    w[:, 0] *= sqrt(0.5)
+    w[0, :] *= sqrt(0.5)
+    w = w[0:basis_size[0], 0:basis_size[1]].transpose().flatten()
     return w
 
+
 w1 = []
 w2 = []
 
-for flo in find_flo(sys.argv[1:]):
-	x,y = load_flo(flo)
+for flo in find_flo(args.files):
+    x, y = load_flo(flo)
     w1.append(get_w(x))
     w2.append(get_w(y))
 
@@ -57,8 +98,10 @@ for i in xrange(len(w1)):
 for i in xrange(len(w2)):
     w2[i] -= w2mean
 
-Q1 = sum([w1[i].reshape(-1,1).dot(w1[i].reshape(1,-1)) for i in xrange(len(w1))]) / len(w1)
-Q2 = sum([w2[i].reshape(-1,1).dot(w2[i].reshape(1,-1)) for i in xrange(len(w2))]) / len(w2)
+Q1 = sum([w1[i].reshape(-1, 1).dot(w1[i].reshape(1, -1))
+          for i in xrange(len(w1))]) / len(w1)
+Q2 = sum([w2[i].reshape(-1, 1).dot(w2[i].reshape(1, -1))
+          for i in xrange(len(w2))]) / len(w2)
 Q1 = np.matrix(Q1)
 Q2 = np.matrix(Q2)
 
@@ -69,11 +112,11 @@ if len(w1) > 1:
             break
         except np.linalg.linalg.LinAlgError:
             mev = min(np.linalg.eig(Q1)[0]).real
-			assert(mev < 0)
+            assert (mev < 0)
             print('Q1', mev)
             if -mev < 1e-6:
                 mev = -1e-6
-			Q1 += (-mev*1.000001) * np.identity(Q1.shape[0])
+            Q1 += (-mev * 1.000001) * np.identity(Q1.shape[0])
 
     while True:
         try:
@@ -81,43 +124,43 @@ if len(w1) > 1:
             break
         except np.linalg.linalg.LinAlgError:
             mev = min(np.linalg.eig(Q2)[0]).real
-			assert(mev < 0)
+            assert (mev < 0)
             print('Q2', mev)
             if -mev < 1e-6:
                 mev = -1e-6
-			Q2 += (-mev*1.000001) * np.identity(Q2.shape[0])
+            Q2 += (-mev * 1.000001) * np.identity(Q2.shape[0])
 else:
     L1 = np.identity(Q1.shape[0])
     L2 = np.identity(Q2.shape[0])
 
-L1 = np.linalg.inv(L1) * lambd
-L2 = np.linalg.inv(L2) * lambd
+L1 = np.linalg.inv(L1) * gamma
+L2 = np.linalg.inv(L2) * gamma
 
-assert(L1.shape == L2.shape)
-assert(L1.shape[0] == L1.shape[1])
+assert (L1.shape == L2.shape)
+assert (L1.shape[0] == L1.shape[1])
 
-f = open('cov.dat', 'wb')
+f = open(args.output, 'wb')
 
 f.write(struct.pack('I', L1.shape[0]))
 f.write(struct.pack('I', L1.shape[1]))
 
 for i in xrange(L1.shape[0]):
     for j in xrange(L1.shape[1]):
-		f.write(struct.pack('f', L1[i,j])) 
+        f.write(struct.pack('f', L1[i, j]))
 
 for i in xrange(L2.shape[0]):
     for j in xrange(L2.shape[1]):
-		f.write(struct.pack('f', L2[i,j]))
+        f.write(struct.pack('f', L2[i, j]))
 
-b1 = L1.dot(w1mean.reshape(-1,1))
-b2 = L2.dot(w2mean.reshape(-1,1))
+b1 = L1.dot(w1mean.reshape(-1, 1))
+b2 = L2.dot(w2mean.reshape(-1, 1))
 
-assert(L1.shape[0] == b1.shape[0])
+assert (L1.shape[0] == b1.shape[0])
 
 for i in xrange(b1.shape[0]):
-	f.write(struct.pack('f', b1[i,0]))
+    f.write(struct.pack('f', b1[i, 0]))
 
 for i in xrange(b2.shape[0]):
-	f.write(struct.pack('f', b2[i,0]))
+    f.write(struct.pack('f', b2[i, 0]))
 
 f.close()

modules/optflow/src/pcaflow.cpp

@@ -48,7 +48,7 @@ namespace cv
 namespace optflow
 {
 
-OpticalFlowPCAFlow::OpticalFlowPCAFlow( const pcaflow::Prior *_prior, const Size _basisSize, float _sparseRate,
+OpticalFlowPCAFlow::OpticalFlowPCAFlow( const PCAPrior *_prior, const Size _basisSize, float _sparseRate,
                                         float _retainedCornersFraction, float _occlusionsThreshold,
                                         float _dampingFactor )
     : prior( _prior ), basisSize( _basisSize ), sparseRate( _sparseRate ),
@@ -60,15 +60,6 @@ OpticalFlowPCAFlow::OpticalFlowPCAFlow( const pcaflow::Prior *_prior, const Size
   CV_Assert( occlusionsThreshold > 0 );
 }
 
-inline float eDistSq( const Point2f &p1, const Point2f &p2 )
-{
-  const float dx = p1.x - p2.x;
-  const float dy = p1.y - p2.y;
-  return dx * dx + dy * dy;
-}
-
-inline float eNormSq( const Point2f &v ) { return v.x * v.x + v.y * v.y; }
-
 template <typename T> static inline int mathSign( T val ) { return ( T( 0 ) < val ) - ( val < T( 0 ) ); }
 
 static inline void symOrtho( double a, double b, double &c, double &s, double &r )
@@ -224,7 +215,7 @@ void OpticalFlowPCAFlow::removeOcclusions( UMat &from, UMat &to, std::vector<Poi
     if ( predictedStatus[i] )
     {
       Point2f flowDiff = features[i] - backwardFeatures[i];
-      if ( eNormSq( flowDiff ) <= threshold )
+      if ( flowDiff.dot( flowDiff ) <= threshold )
       {
         features[j] = features[i];
         predictedFeatures[j] = predictedFeatures[i];
@@ -462,10 +453,7 @@ void OpticalFlowPCAFlow::collectGarbage() {}
 
 Ptr<DenseOpticalFlow> createOptFlow_PCAFlow() { return makePtr<OpticalFlowPCAFlow>(); }
 
-namespace pcaflow
-{
-
-Prior::Prior( const char *pathToPrior )
+PCAPrior::PCAPrior( const char *pathToPrior )
 {
   FILE *f = fopen( pathToPrior, "r" );
   CV_Assert( f );
@@ -487,7 +475,7 @@ Prior::Prior( const char *pathToPrior )
   fclose( f );
 }
 
-void Prior::fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const
+void PCAPrior::fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const
 {
   memcpy( A1, L1.ptr<float>(), L1.size().area() * sizeof( float ) );
   memcpy( A2, L2.ptr<float>(), L2.size().area() * sizeof( float ) );
@@ -496,4 +484,3 @@ void Prior::fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const
 }
 }
 }
-}