Merge pull request #4 from lenlen/test_perftest

Added perf_test and test based on distance and overlap for tracking API

modules/tracking/perf/perf_Tracker.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "perf_precomp.hpp"
+#include <fstream>
+
+using namespace std;
+using namespace cv;
+using namespace perf;
+using std::tr1::make_tuple;
+using std::tr1::get;
+
+//write sanity: ./bin/opencv_perf_tracking --perf_write_sanity=true --perf_min_samples=1
+//verify sanity: ./bin/opencv_perf_tracking --perf_min_samples=1
+
+#define TESTSET_NAMES testing::Values("david","dudek","faceocc2")
+//#define TESTSET_NAMES testing::internal::ValueArray1<string>("david")
+#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+
+const string TRACKING_DIR = "cv/tracking";
+const string FOLDER_IMG = "data";
+
+typedef perf::TestBaseWithParam<tr1::tuple<string, int> > tracking;
+
+std::vector<std::string> splitString( std::string s, std::string delimiter )
+{
+  std::vector<string> token;
+  size_t pos = 0;
+  while ( ( pos = s.find( delimiter ) ) != std::string::npos )
+  {
+    token.push_back( s.substr( 0, pos ) );
+    s.erase( 0, pos + delimiter.length() );
+  }
+  token.push_back( s );
+  return token;
+}
+
+void checkData( const string& datasetMeta, int& startFrame, string& prefix, string& suffix )
+{
+  //get informations on the current test data
+  FileStorage fs;
+  fs.open( datasetMeta, FileStorage::READ );
+  fs["start"] >> startFrame;
+  fs["prefix"] >> prefix;
+  fs["suffix"] >> suffix;
+  fs.release();
+}
+
+bool getGroundTruth( const string& gtFile, vector<Rect>& gtBBs )
+{
+  ifstream gt;
+  //open the ground truth
+  gt.open( gtFile.c_str() );
+  if( !gt.is_open() )
+  {
+    return false;
+  }
+  string line;
+  Rect currentBB;
+  while ( getline( gt, line ) )
+  {
+    vector<string> tokens = splitString( line, "," );
+
+    if( tokens.size() != 4 )
+    {
+      return false;
+    }
+
+    gtBBs.push_back(
+        Rect( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ) );
+  }
+  return true;
+}
+
+void getSegment( int segmentId, int numSegments, int bbCounter, int& startFrame, int& endFrame )
+{
+  //compute the start and the and for each segment
+  int gtStartFrame = startFrame;
+  int numFrame = bbCounter / numSegments;
+  startFrame += ( segmentId - 1 ) * numFrame;
+  endFrame = startFrame + numFrame;
+
+  if( ( segmentId ) == numSegments )
+    endFrame = bbCounter + gtStartFrame - 1;
+}
+
+void getMatOfRects( const vector<Rect>& bbs, Mat& bbs_mat )
+{
+  for ( size_t b = 0; b < bbs.size(); b++ )
+  {
+    bbs_mat.at<float>( b, 0 ) = bbs[b].x;
+    bbs_mat.at<float>( b, 1 ) = bbs[b].y;
+    bbs_mat.at<float>( b, 2 ) = bbs[b].width;
+    bbs_mat.at<float>( b, 3 ) = bbs[b].height;
+  }
+}
+
+PERF_TEST_P(tracking, mil, testing::Combine(TESTSET_NAMES, SEGMENTS))
+{
+  string video = get<0>( GetParam() );
+  int segmentId = get<1>( GetParam() );
+
+  int startFrame;
+  string prefix;
+  string suffix;
+  string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" );
+  checkData( datasetMeta, startFrame, prefix, suffix );
+  int gtStartFrame = startFrame;
+
+  vector<Rect> gtBBs;
+  string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" );
+  if( !getGroundTruth( gtFile, gtBBs ) )
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  int bbCounter = gtBBs.size();
+
+  Mat frame;
+  bool initialized = false;
+  vector<Rect> bbs;
+
+  Ptr<Tracker> tracker = Tracker::create( "MIL" );
+  string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+  int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) );
+  int endFrame = 0;
+  getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame );
+
+  Rect currentBB = gtBBs[startFrame - gtStartFrame];
+
+  TEST_CYCLE_N(1)
+  {
+    VideoCapture c;
+    c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) );
+    c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+    for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+    {
+      c >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
+      if( !initialized )
+      {
+        if( !tracker->init( frame, currentBB ) )
+        {
+          FAIL()<< "Could not initialize tracker" << endl;
+          return;
+        }
+        initialized = true;
+      }
+      else if( initialized )
+      {
+        tracker->update( frame, currentBB );
+      }
+      bbs.push_back( currentBB );
+
+    }
+  }
+  //save the bounding boxes in a Mat
+  Mat bbs_mat( bbs.size(), 4, CV_32F );
+  getMatOfRects( bbs, bbs_mat );
+
+  SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE );
+
+}
+
+PERF_TEST_P(tracking, boosting, testing::Combine(TESTSET_NAMES, SEGMENTS))
+{
+  string video = get<0>( GetParam() );
+  int segmentId = get<1>( GetParam() );
+
+  int startFrame;
+  string prefix;
+  string suffix;
+  string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" );
+  checkData( datasetMeta, startFrame, prefix, suffix );
+  int gtStartFrame = startFrame;
+
+  vector<Rect> gtBBs;
+  string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" );
+  if( !getGroundTruth( gtFile, gtBBs ) )
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  int bbCounter = gtBBs.size();
+
+  Mat frame;
+  bool initialized = false;
+  vector<Rect> bbs;
+
+  Ptr<Tracker> tracker = Tracker::create( "BOOSTING" );
+  string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+  int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) );
+  int endFrame = 0;
+  getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame );
+
+  Rect currentBB = gtBBs[startFrame - gtStartFrame];
+
+  TEST_CYCLE_N(1)
+  {
+    VideoCapture c;
+    c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) );
+    c.set( CAP_PROP_POS_FRAMES, startFrame );
+    for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+    {
+      c >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
+      if( !initialized )
+      {
+        if( !tracker->init( frame, currentBB ) )
+        {
+          FAIL()<< "Could not initialize tracker" << endl;
+          return;
+        }
+        initialized = true;
+      }
+      else if( initialized )
+      {
+        tracker->update( frame, currentBB );
+      }
+      bbs.push_back( currentBB );
+
+    }
+  }
+  //save the bounding boxes in a Mat
+  Mat bbs_mat( bbs.size(), 4, CV_32F );
+  getMatOfRects( bbs, bbs_mat );
+
+  SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE );
+
+}

modules/tracking/perf/perf_tracking.cpp

-/*M///////////////////////////////////////////////////////////////////////////////////////
- //
- //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
- //
- //  By downloading, copying, installing or using the software you agree to this license.
- //  If you do not agree to this license, do not download, install,
- //  copy or use the software.
- //
- //
- //                           License Agreement
- //                For Open Source Computer Vision Library
- //
- // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
- // Third party copyrights are property of their respective owners.
- //
- // Redistribution and use in source and binary forms, with or without modification,
- // are permitted provided that the following conditions are met:
- //
- //   * Redistribution's of source code must retain the above copyright notice,
- //     this list of conditions and the following disclaimer.
- //
- //   * Redistribution's in binary form must reproduce the above copyright notice,
- //     this list of conditions and the following disclaimer in the documentation
- //     and/or other materials provided with the distribution.
- //
- //   * The name of the copyright holders may not be used to endorse or promote products
- //     derived from this software without specific prior written permission.
- //
- // This software is provided by the copyright holders and contributors "as is" and
- // any express or implied warranties, including, but not limited to, the implied
- // warranties of merchantability and fitness for a particular purpose are disclaimed.
- // In no event shall the Intel Corporation or contributors be liable for any direct,
- // indirect, incidental, special, exemplary, or consequential damages
- // (including, but not limited to, procurement of substitute goods or services;
- // loss of use, data, or profits; or business interruption) however caused
- // and on any theory of liability, whether in contract, strict liability,
- // or tort (including negligence or otherwise) arising in any way out of
- // the use of this software, even if advised of the possibility of such damage.
- //
- //M*/
-
-#include "perf_precomp.hpp"
-
-using namespace std;
-using namespace cv;
-using namespace perf;

modules/tracking/samples/tracker.cpp

@@ -13,15 +13,17 @@ static bool selectObject = false;
 static bool startSelection = false;
 
 static const char* keys =
-{ "{@tracker_algorithm | | tracker algorithm }"
-    "{@video_name        | | video name        }" };
+{ "{@tracker_algorithm | | Tracker algorithm }"
+    "{@video_name      | | video name        }"
+    "{@start_frame     |1| Start frame       }" };
 
 static void help()
 {
   cout << "\nThis example shows the functionality of \"Long-term optical tracking API\""
        "-- pause video [p] and draw a bounding box around the target to start the tracker\n"
+       "Example of <video_name> is in opencv_extra/testdata/cv/tracking/\n"
        "Call:\n"
-       "./tracker <tracker_algorithm> <video_name>\n"
+       "./tracker <tracker_algorithm> <video_name> <start_frame>\n"
        << endl;
 
   cout << "\n\nHot keys: \n"
@@ -69,6 +71,7 @@ int main( int argc, char** argv )
 
   String tracker_algorithm = parser.get<String>( 0 );
   String video_name = parser.get<String>( 1 );
+  int start_frame = parser.get<int>( 2 );
 
   if( tracker_algorithm.empty() || video_name.empty() )
   {
@@ -79,6 +82,7 @@ int main( int argc, char** argv )
   //open the capture
   VideoCapture cap;
   cap.open( video_name );
+  cap.set( CAP_PROP_POS_FRAMES, start_frame );
 
   if( !cap.isOpened() )
   {
@@ -108,11 +112,19 @@ int main( int argc, char** argv )
   imshow( "Tracking API", image );
 
   bool initialized = false;
+  int frameCounter = 0;
+
   for ( ;; )
   {
     if( !paused )
     {
       cap >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
       frame.copyTo( image );
 
       if( !initialized && selectObject )
@@ -134,6 +146,7 @@ int main( int argc, char** argv )
         }
       }
       imshow( "Tracking API", image );
+      frameCounter++;
     }
 
     char c = (char) waitKey( 2 );

modules/tracking/src/trackerBoosting.cpp

@@ -56,7 +56,7 @@ TrackerBoosting::Params::Params()
 {
   numClassifiers = 100;
   samplerOverlap = 0.99f;
-  samplerSearchFactor = 2;
+  samplerSearchFactor = 1.8;
   iterationInit = 50;
   featureSetNumFeatures = ( numClassifiers * 10 ) + iterationInit;
 }
@@ -108,6 +108,7 @@ void TrackerBoosting::write( cv::FileStorage& fs ) const
 
 bool TrackerBoosting::initImpl( const Mat& image, const Rect& boundingBox )
 {
+  srand (1);
   //sampling
   Mat_<int> intImage;
   Mat_<double> intSqImage;

modules/tracking/src/trackerBoostingModel.hpp

@@ -99,7 +99,6 @@ class TrackerBoostingModel : public TrackerModel
  private:
 
   std::vector<Mat> currentSample;
-  std::vector<std::pair<float, float> > meanSigmaPair;
 
   int mode;
 };

modules/tracking/src/trackerMIL.cpp

@@ -124,6 +124,7 @@ void TrackerMIL::compute_integral( const Mat & img, Mat & ii_img )
 
 bool TrackerMIL::initImpl( const Mat& image, const Rect& boundingBox )
 {
+  srand (1);
   Mat intImage;
   compute_integral( image, intImage );
   TrackerSamplerCSC::Params CSCparameters;

modules/tracking/test/test_tracker.cpp

-/*M///////////////////////////////////////////////////////////////////////////////////////
- //
- //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
- //
- //  By downloading, copying, installing or using the software you agree to this license.
- //  If you do not agree to this license, do not download, install,
- //  copy or use the software.
- //
- //
- //                           License Agreement
- //                For Open Source Computer Vision Library
- //
- // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
- // Third party copyrights are property of their respective owners.
- //
- // Redistribution and use in source and binary forms, with or without modification,
- // are permitted provided that the following conditions are met:
- //
- //   * Redistribution's of source code must retain the above copyright notice,
- //     this list of conditions and the following disclaimer.
- //
- //   * Redistribution's in binary form must reproduce the above copyright notice,
- //     this list of conditions and the following disclaimer in the documentation
- //     and/or other materials provided with the distribution.
- //
- //   * The name of the copyright holders may not be used to endorse or promote products
- //     derived from this software without specific prior written permission.
- //
- // This software is provided by the copyright holders and contributors "as is" and
- // any express or implied warranties, including, but not limited to, the implied
- // warranties of merchantability and fitness for a particular purpose are disclaimed.
- // In no event shall the Intel Corporation or contributors be liable for any direct,
- // indirect, incidental, special, exemplary, or consequential damages
- // (including, but not limited to, procurement of substitute goods or services;
- // loss of use, data, or profits; or business interruption) however caused
- // and on any theory of liability, whether in contract, strict liability,
- // or tort (including negligence or otherwise) arising in any way out of
- // the use of this software, even if advised of the possibility of such damage.
- //
- //M*/
-
-#include "test_precomp.hpp"
-#include "opencv2/tracking.hpp"
-
-using namespace cv;
-using namespace std;
-
-class CV_TrackerBaseTest : public cvtest::BaseTest
-{
- public:
-  CV_TrackerBaseTest();
-  virtual ~CV_TrackerBaseTest();
-
-};
-
-CV_TrackerBaseTest::CV_TrackerBaseTest()
-{
-
-}
-
-CV_TrackerBaseTest::~CV_TrackerBaseTest()
-{
-
-}
-
-/************************************ TrackerMIL ************************************/
-
-class CV_TrackerMILTest : public CV_TrackerBaseTest
-{
- public:
-  CV_TrackerMILTest();
-  ~CV_TrackerMILTest();
-
- protected:
-  void run( int );
-};
-
-CV_TrackerMILTest::CV_TrackerMILTest()
-{
-}
-
-CV_TrackerMILTest::~CV_TrackerMILTest()
-{
-}
-
-void CV_TrackerMILTest::run( int )
-{
-  ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC );
-  ts->printf( cvtest::TS::LOG, "CV_TrackerMILTest to be implemented" );
-}
-
-TEST(DISABLED_Tracking_TrackerMIL, accuracy)
-{
-  CV_TrackerMILTest test;
-  test.safe_run();
-}
-
-/************************************ TrackerBoosting ************************************/
-
-class CV_TrackerBoostingTest : public CV_TrackerBaseTest
-{
- public:
-  CV_TrackerBoostingTest();
-  ~CV_TrackerBoostingTest();
-
- protected:
-  void run( int );
-};
-
-CV_TrackerBoostingTest::CV_TrackerBoostingTest()
-{
-}
-
-CV_TrackerBoostingTest::~CV_TrackerBoostingTest()
-{
-}
-
-void CV_TrackerBoostingTest::run( int )
-{
-  ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC );
-  ts->printf( cvtest::TS::LOG, "CV_TrackerBoostingTest to be implemented" );
-}
-
-TEST(DISABLED_Tracking_TrackerBoosting, accuracy)
-{
-  CV_TrackerBoostingTest test;
-  test.safe_run();
-}
-
-/* End of file. */

modules/tracking/test/test_trackerOPE.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "test_precomp.hpp"
+#include "opencv2/tracking.hpp"
+#include <fstream>
+
+using namespace cv;
+using namespace testing;
+using namespace std;
+
+#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > >
+#define GET_PARAM(k) std::tr1::get< k >(GetParam())
+#define TESTSET_NAMES testing::Values("david","dudek","faceocc2")
+#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50)
+#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1)
+
+const string TRACKING_DIR = "tracking";
+const string FOLDER_IMG = "data";
+
+/*
+ * The Evaluation Methodologies are partially based on:
+ * ====================================================================================================================
+ *  [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013
+ *
+ */
+
+//Robustness Evaluation, see [OTB] chapter 4. OPE One Pass Evaluation
+class TrackerOPETest
+{
+ public:
+  enum
+  {
+    DISTANCE = 1,  // test trackers based on euclidean distance
+    OVERLAP = 2    // test trackers based on the overlapping ratio
+  };
+
+  TrackerOPETest( Ptr<Tracker> _tracker, int _testType, string _video, float _threshold );
+  virtual ~TrackerOPETest();
+  virtual void run();
+  string getRatioSucc() const;
+
+ protected:
+  void checkDataTest();
+
+  void distanceTest();
+  void overlapTest();
+
+  Ptr<Tracker> tracker;
+  int testType;
+  string video;
+  std::vector<Rect> bbs;
+  int startFrame;
+  string suffix;
+  string prefix;
+  float threshold;
+  float ratioSucc;
+
+ private:
+  float calcDistance( Rect a, Rect b );
+  float calcOverlap( Rect a, Rect b );
+  std::vector<std::string> splitString( std::string s, std::string delimiter );
+
+};
+
+TrackerOPETest::TrackerOPETest( Ptr<Tracker> _tracker, int _testType, string _video, float _threshold ) :
+    tracker( _tracker ),
+    testType( _testType ),
+    video( _video ),
+    threshold( _threshold )
+{
+  startFrame = 1;
+  ratioSucc = 0;
+}
+
+TrackerOPETest::~TrackerOPETest()
+{
+
+}
+
+string TrackerOPETest::getRatioSucc() const
+{
+  stringstream ratio;
+  ratio << ratioSucc;
+  return ratio.str();
+}
+
+std::vector<std::string> TrackerOPETest::splitString( std::string s, std::string delimiter )
+{
+  std::vector<string> token;
+  size_t pos = 0;
+  while ( ( pos = s.find( delimiter ) ) != std::string::npos )
+  {
+    token.push_back( s.substr( 0, pos ) );
+    s.erase( 0, pos + delimiter.length() );
+  }
+  token.push_back( s );
+  return token;
+}
+
+float TrackerOPETest::calcDistance( Rect a, Rect b )
+{
+  Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 );
+  Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 );
+  return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) );
+}
+
+float TrackerOPETest::calcOverlap( Rect a, Rect b )
+{
+  float aArea = a.width * a.height;
+  float bArea = b.width * b.height;
+
+  if( aArea < bArea )
+  {
+    a.x -= ( b.width - a.width ) / 2;
+    a.y -= ( b.height - a.height ) / 2;
+    a.width = b.width;
+    a.height = b.height;
+  }
+  else
+  {
+    b.x -= ( a.width - b.width ) / 2;
+    b.y -= ( a.height - b.height ) / 2;
+    b.width = a.width;
+    b.height = a.height;
+  }
+
+  Rect rectIntersection = a & b;
+  Rect rectUnion = a | b;
+  float iArea = rectIntersection.width * rectIntersection.height;
+  float uArea = rectUnion.width * rectUnion.height;
+  float overlap = iArea / uArea;
+  return overlap;
+}
+
+void TrackerOPETest::distanceTest()
+{
+  Mat frame;
+  bool initialized = false;
+
+  Rect currentBB = bbs.at( 0 );
+  float sumDistance = 0;
+  int frameCounter = 0;
+  int frameCounterSucc = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+  for ( ;; )
+  {
+    c >> frame;
+
+    if( frame.empty() )
+    {
+      break;
+    }
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+    float curDistance = calcDistance( currentBB, bbs.at( frameCounter ) );
+    if( curDistance <= threshold )
+      frameCounterSucc++;
+    sumDistance += curDistance;
+    frameCounter++;
+  }
+
+  float distance = sumDistance / frameCounter;
+  ratioSucc = (float) frameCounterSucc / (float) frameCounter;
+
+  if( distance > threshold )
+  {
+    FAIL()<< "Incorrect distance: curr = " << distance << ", max = " << threshold << endl;
+    return;
+  }
+
+}
+
+void TrackerOPETest::overlapTest()
+{
+  Mat frame;
+  bool initialized = false;
+  Rect currentBB = bbs.at( 0 );
+  float sumOverlap = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  int frameCounter = 0;
+  int frameCounterSucc = 0;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+  for ( ;; )
+  {
+    c >> frame;
+    if( frame.empty() )
+    {
+      break;
+    }
+
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+    float curOverlap = calcOverlap( currentBB, bbs.at( frameCounter ) );
+    if( curOverlap >= threshold )
+      frameCounterSucc++;
+
+    sumOverlap += curOverlap;
+    frameCounter++;
+  }
+
+  float overlap = sumOverlap / frameCounter;
+  ratioSucc = (float) frameCounterSucc / (float) frameCounter;
+
+  if( overlap < threshold )
+  {
+    FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl;
+    return;
+  }
+}
+
+void TrackerOPETest::checkDataTest()
+{
+  string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt";
+
+  ifstream gt;
+  //open the ground truth
+  gt.open( gtFile.c_str() );
+  if( !gt.is_open() )
+  {
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  }
+  string line;
+  int bbCounter = 0;
+  while ( getline( gt, line ) )
+  {
+    vector<string> tokens = splitString( line, "," );
+    Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) );
+    if( tokens.size() != 4 )
+    {
+      FAIL()<< "Incorrect ground truth file";
+    }
+    bbs.push_back( bb );
+    bbCounter++;
+  }
+
+  FileStorage fs;
+  fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ );
+  fs["start"] >> startFrame;
+  fs["prefix"] >> prefix;
+  fs["suffix"] >> suffix;
+  fs.release();
+
+}
+
+void TrackerOPETest::run()
+{
+  srand( 1 );
+
+  SCOPED_TRACE( "A" );
+
+  if( !tracker )
+  {
+    FAIL()<< "Error in the instantiation of the tracker" << endl;
+    return;
+  }
+
+  checkDataTest();
+
+  //check for failure
+  if( ::testing::Test::HasFatalFailure() )
+    return;
+
+  if( testType == DISTANCE )
+  {
+    distanceTest();
+  }
+  else if( testType == OVERLAP )
+  {
+    overlapTest();
+  }
+  else
+  {
+    FAIL()<< "Test type unknown" << endl;
+    return;
+  }
+
+}
+
+/****************************************************************************************\
+*                                Tests registrations                                     *
+ \****************************************************************************************/
+
+//[TESTDATA] [LOCATION ERROR THRESHOLD]
+PARAM_TEST_CASE(OPE_Distance, string, float)
+{
+  string dataset;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    threshold = GET_PARAM(1);
+  }
+};
+
+//[TESTDATA] [OVERLAP THRESHOLD]
+PARAM_TEST_CASE(OPE_Overlap, string, float)
+{
+  string dataset;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    threshold = GET_PARAM(1);
+  }
+};
+
+TEST_P(OPE_Distance, MIL)
+{
+  TrackerOPETest test( Tracker::create( "MIL" ), TrackerOPETest::DISTANCE, dataset, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(OPE_Overlap, MIL)
+{
+  TrackerOPETest test( Tracker::create( "MIL" ), TrackerOPETest::OVERLAP, dataset, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(OPE_Distance, Boosting)
+{
+  TrackerOPETest test( Tracker::create( "BOOSTING" ), TrackerOPETest::DISTANCE, dataset, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(OPE_Overlap, Boosting)
+{
+  TrackerOPETest test( Tracker::create( "BOOSTING" ), TrackerOPETest::OVERLAP, dataset, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+INSTANTIATE_TEST_CASE_P( Tracking, OPE_Distance, testing::Combine( TESTSET_NAMES, LOCATION_ERROR_THRESHOLD ) );
+
+INSTANTIATE_TEST_CASE_P( Tracking, OPE_Overlap, testing::Combine( TESTSET_NAMES, OVERLAP_THRESHOLD ) );
+
+/* End of file. */

modules/tracking/test/test_trackerSRE.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "test_precomp.hpp"
+#include "opencv2/tracking.hpp"
+#include <fstream>
+
+using namespace cv;
+using namespace testing;
+using namespace std;
+
+#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > >
+#define GET_PARAM(k) std::tr1::get< k >(GetParam())
+#define TESTSET_NAMES testing::Values("david","dudek","faceocc2")
+#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50)
+#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1)
+
+#define SPATIAL_SHIFTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10 , 11, 12)
+
+const string TRACKING_DIR = "tracking";
+const string FOLDER_IMG = "data";
+
+/*
+ * The Evaluation Methodologies are partially based on:
+ * ====================================================================================================================
+ *  [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013
+ *
+ */
+
+//Robustness Evaluation, see [OTB] chapter 4. SRE Spatial robustness evaluation
+class TrackerSRETest
+{
+ public:
+  enum
+  {
+    DISTANCE = 1,  // test trackers based on euclidean distance
+    OVERLAP = 2    // test trackers based on the overlapping ratio
+  };
+
+  TrackerSRETest( const Ptr<Tracker> _tracker, int _testType, string _video, int _shift, float _threshold );
+  virtual ~TrackerSRETest();
+  virtual void run();
+  string getRatioSucc() const;
+
+ protected:
+  void checkDataTest();
+
+  void distanceTest();
+  void overlapTest();
+
+  Ptr<Tracker> tracker;
+  int testType;
+  string video;
+  std::vector<Rect> bbs;
+  int startFrame;
+  string suffix;
+  string prefix;
+  float threshold;
+  int shift;
+  float ratioSucc;
+
+ private:
+  float calcDistance( Rect a, Rect b );
+  float calcOverlap( Rect a, Rect b );
+  std::vector<std::string> splitString( std::string s, std::string delimiter );
+
+};
+
+TrackerSRETest::TrackerSRETest( const Ptr<Tracker> _tracker, int _testType, string _video, int _shift, float _threshold ) :
+    tracker( _tracker ),
+    testType( _testType ),
+    video( _video ),
+    threshold( _threshold ),
+    shift( _shift )
+{
+  startFrame = 1;
+  ratioSucc = 0;
+}
+
+TrackerSRETest::~TrackerSRETest()
+{
+
+}
+
+string TrackerSRETest::getRatioSucc() const
+{
+  stringstream ratio;
+  ratio << ratioSucc;
+  return ratio.str();
+}
+
+std::vector<std::string> TrackerSRETest::splitString( std::string s, std::string delimiter )
+{
+  std::vector<string> token;
+  size_t pos = 0;
+  while ( ( pos = s.find( delimiter ) ) != std::string::npos )
+  {
+    token.push_back( s.substr( 0, pos ) );
+    s.erase( 0, pos + delimiter.length() );
+  }
+  token.push_back( s );
+  return token;
+}
+
+float TrackerSRETest::calcDistance( Rect a, Rect b )
+{
+  Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 );
+  Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 );
+  return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) );
+}
+
+float TrackerSRETest::calcOverlap( Rect a, Rect b )
+{
+  float aArea = a.width * a.height;
+  float bArea = b.width * b.height;
+
+  if( aArea < bArea )
+  {
+    a.x -= ( b.width - a.width ) / 2;
+    a.y -= ( b.height - a.height ) / 2;
+    a.width = b.width;
+    a.height = b.height;
+  }
+  else
+  {
+    b.x -= ( a.width - b.width ) / 2;
+    b.y -= ( a.height - b.height ) / 2;
+    b.width = a.width;
+    b.height = a.height;
+  }
+
+  Rect rectIntersection = a & b;
+  Rect rectUnion = a | b;
+  float iArea = rectIntersection.width * rectIntersection.height;
+  float uArea = rectUnion.width * rectUnion.height;
+  float overlap = iArea / uArea;
+  return overlap;
+}
+
+void TrackerSRETest::distanceTest()
+{
+  Mat frame;
+  bool initialized = false;
+
+  Rect currentBB = bbs.at( 0 );
+  float sumDistance = 0;
+  int frameCounter = 0;
+  int frameCounterSucc = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+  for ( ;; )
+  {
+    c >> frame;
+    if( frame.empty() )
+    {
+      break;
+    }
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+
+    float curDistance = calcDistance( currentBB, bbs.at( frameCounter ) );
+    if( curDistance <= threshold )
+      frameCounterSucc++;
+    sumDistance += curDistance;
+    frameCounter++;
+  }
+
+  float distance = sumDistance / frameCounter;
+  ratioSucc = (float) frameCounterSucc / (float) frameCounter;
+
+  if( distance > threshold )
+  {
+    FAIL()<< "Incorrect distance: curr = " << distance << ", max = " << threshold << endl;
+    return;
+  }
+
+}
+
+void TrackerSRETest::overlapTest()
+{
+  Mat frame;
+  bool initialized = false;
+  Rect currentBB = bbs.at( 0 );
+  float sumOverlap = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  int frameCounter = 0;
+  int frameCounterSucc = 0;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+  for ( ;; )
+  {
+    c >> frame;
+    if( frame.empty() )
+    {
+      break;
+    }
+
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+    float curOverlap = calcOverlap( currentBB, bbs.at( frameCounter ) );
+    if( curOverlap >= threshold )
+      frameCounterSucc++;
+
+    sumOverlap += curOverlap;
+    frameCounter++;
+  }
+
+  float overlap = sumOverlap / frameCounter;
+  ratioSucc = (float) frameCounterSucc / (float) frameCounter;
+
+  if( overlap < threshold )
+  {
+    FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl;
+    return;
+  }
+}
+
+void TrackerSRETest::checkDataTest()
+{
+  string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt";
+
+  ifstream gt;
+  //open the ground truth
+  gt.open( gtFile.c_str() );
+  if( !gt.is_open() )
+  {
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  }
+  string line;
+  int bbCounter = 0;
+  while ( getline( gt, line ) )
+  {
+    vector<string> tokens = splitString( line, "," );
+    Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) );
+    if( tokens.size() != 4 )
+    {
+      FAIL()<< "Incorrect ground truth file";
+    }
+
+    //apply the shift
+    if( bbCounter == 0 )
+    {
+      Point center( bb.x + ( bb.width / 2 ), bb.y + ( bb.height / 2 ) );
+
+      int xLimit = bb.x + bb.width - 1;
+      int yLimit = bb.y + bb.height - 1;
+
+      int h = 0;
+      int w = 0;
+      float ratio = 1.0;
+
+      switch ( shift )
+      {
+        case 1:
+          //center shift left
+          bb.x = bb.x - ceil( 0.1 * bb.width );
+          break;
+        case 2:
+          //center shift right
+          bb.x = bb.x + ceil( 0.1 * bb.width );
+          break;
+        case 3:
+          //center shift up
+          bb.y = bb.y - ceil( 0.1 * bb.height );
+          break;
+        case 4:
+          //center shift down
+          bb.y = bb.y + ceil( 0.1 * bb.height );
+          break;
+        case 5:
+          //corner shift top-left
+          bb.x = round( bb.x - ( 0.1 * bb.width ) );
+          bb.y = round( bb.y - ( 0.1 * bb.height ) );
+
+          bb.width = xLimit - bb.x + 1;
+          bb.height = yLimit - bb.y + 1;
+          break;
+        case 6:
+          //corner shift top-right
+          xLimit = round( xLimit + 0.1 * bb.width );
+
+          bb.y = round( bb.y - ( 0.1 * bb.height ) );
+          bb.width = xLimit - bb.x + 1;
+          bb.height = yLimit - bb.y + 1;
+          break;
+        case 7:
+          //corner shift bottom-left
+          bb.x = round( bb.x - ( 0.1 * bb.width ) );
+          yLimit = round( yLimit + 0.1 * bb.height );
+
+          bb.width = xLimit - bb.x + 1;
+          bb.height = yLimit - bb.y + 1;
+          break;
+        case 8:
+          //corner shift bottom-right
+          xLimit = round( xLimit + 0.1 * bb.width );
+          yLimit = round( yLimit + 0.1 * bb.height );
+
+          bb.width = xLimit - bb.x + 1;
+          bb.height = yLimit - bb.y + 1;
+          break;
+        case 9:
+          //scale 0.8
+          ratio = 0.8;
+          w = ratio * bb.width;
+          h = ratio * bb.height;
+
+          bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h );
+          break;
+        case 10:
+          //scale 0.9
+          ratio = 0.9;
+          w = ratio * bb.width;
+          h = ratio * bb.height;
+
+          bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h );
+          break;
+        case 11:
+          //scale 1.1
+          ratio = 1.1;
+          w = ratio * bb.width;
+          h = ratio * bb.height;
+
+          bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h );
+          break;
+        case 12:
+          //scale 1.2
+          ratio = 1.2;
+          w = ratio * bb.width;
+          h = ratio * bb.height;
+
+          bb = Rect( center.x - ( w / 2 ), center.y - ( h / 2 ), w, h );
+          break;
+        default:
+          break;
+      }
+    }
+    bbs.push_back( bb );
+    bbCounter++;
+  }
+
+  FileStorage fs;
+  fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ );
+  fs["start"] >> startFrame;
+  fs["prefix"] >> prefix;
+  fs["suffix"] >> suffix;
+  fs.release();
+
+}
+
+void TrackerSRETest::run()
+{
+  srand( 1 );
+  SCOPED_TRACE( "A" );
+
+  if( !tracker )
+  {
+    FAIL()<< "Error in the instantiation of the tracker" << endl;
+    return;
+  }
+
+  checkDataTest();
+
+  //check for failure
+  if( ::testing::Test::HasFatalFailure() )
+    return;
+
+  if( testType == DISTANCE )
+  {
+    distanceTest();
+  }
+  else if( testType == OVERLAP )
+  {
+    overlapTest();
+  }
+  else
+  {
+    FAIL()<< "Test type unknown" << endl;
+    return;
+  }
+
+}
+
+/****************************************************************************************\
+*                                Tests registrations                                     *
+ \****************************************************************************************/
+
+//[TESTDATA] [#SHIFT] [LOCATION ERROR THRESHOLD]
+PARAM_TEST_CASE(SRE_Distance, string, int, float)
+{
+  string dataset;
+  int shift;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    shift = GET_PARAM(1);
+    threshold = GET_PARAM(2);
+  }
+};
+
+//[TESTDATA] [#SHIFT] [OVERLAP THRESHOLD]
+PARAM_TEST_CASE(SRE_Overlap, string, int, float)
+{
+  string dataset;
+  int shift;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    shift = GET_PARAM(1);
+    threshold = GET_PARAM(2);
+  }
+};
+
+TEST_P(SRE_Distance, MIL)
+{
+  TrackerSRETest test( Tracker::create( "MIL" ), TrackerSRETest::DISTANCE, dataset, shift, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(SRE_Overlap, MIL)
+{
+  TrackerSRETest test( Tracker::create( "MIL" ), TrackerSRETest::OVERLAP, dataset, shift, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(SRE_Distance, Boosting)
+{
+  TrackerSRETest test( Tracker::create( "BOOSTING" ), TrackerSRETest::DISTANCE, dataset, shift, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(SRE_Overlap, Boosting)
+{
+  TrackerSRETest test( Tracker::create( "BOOSTING" ), TrackerSRETest::OVERLAP, dataset, shift, threshold );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+INSTANTIATE_TEST_CASE_P( Tracking, SRE_Distance, testing::Combine( TESTSET_NAMES, SPATIAL_SHIFTS, LOCATION_ERROR_THRESHOLD ) );
+
+INSTANTIATE_TEST_CASE_P( Tracking, SRE_Overlap, testing::Combine( TESTSET_NAMES, SPATIAL_SHIFTS, OVERLAP_THRESHOLD ) );
+
+/* End of file. */

modules/tracking/test/test_trackerTRE.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "test_precomp.hpp"
+#include "opencv2/tracking.hpp"
+#include <fstream>
+#include <algorithm>
+
+using namespace cv;
+using namespace testing;
+using namespace std;
+
+#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > >
+#define GET_PARAM(k) std::tr1::get< k >(GetParam())
+#define TESTSET_NAMES testing::Values("david","dudek","faceocc2")
+#define LOCATION_ERROR_THRESHOLD testing::Values(0, 10, 20, 30, 40, 50)
+#define OVERLAP_THRESHOLD testing::Values(0, 0.2, 0.4, 0.6, 0.8, 1)
+//Fixed sampling on the images sequence
+#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+
+const string TRACKING_DIR = "tracking";
+const string FOLDER_IMG = "data";
+const string FOLDER_OMIT_INIT = "initOmit";
+
+/*
+ * The Evaluation Methodologies are partially based on:
+ * ====================================================================================================================
+ *  [OTB] Y. Wu, J. Lim, and M.-H. Yang, "Online object tracking: A benchmark," in Computer Vision and Pattern Recognition (CVPR), 2013
+ *
+ */
+
+//Robustness Evaluation, see [OTB] chapter 4. temporal robustness evaluation
+//each sequence is partitioned into 10 (fixed) segments, slight change respect to [OTB]
+class TrackerTRETest
+{
+ public:
+  enum
+  {
+    DISTANCE = 1,  // test trackers based on euclidean distance
+    OVERLAP = 2    // test trackers based on the overlapping ratio
+  };
+
+  TrackerTRETest( const Ptr<Tracker> _tracker, int _testType, string _video, float _threshold, int _segmentIdx );
+  virtual ~TrackerTRETest();
+  virtual void run();
+  string getRatioSucc() const;
+
+ protected:
+  void checkDataTest();
+
+  void distanceTest();
+  void overlapTest();
+
+  Ptr<Tracker> tracker;
+  int testType;
+  string video;
+  std::vector<Rect> bbs;
+  int gtStartFrame;
+  int startFrame;
+  int endFrame;
+  string suffix;
+  string prefix;
+  float threshold;
+  int segmentIdx;
+  vector<int> validSequence;
+  float ratioSucc;
+
+ private:
+  float calcDistance( Rect a, Rect b );
+  float calcOverlap( Rect a, Rect b );
+  std::vector<std::string> splitString( std::string s, std::string delimiter );
+
+};
+
+TrackerTRETest::TrackerTRETest( const Ptr<Tracker> _tracker, int _testType, string _video, float _threshold, int _segmentIdx ) :
+    tracker( _tracker ),
+    testType( _testType ),
+    video( _video ),
+    threshold( _threshold ),
+    segmentIdx( _segmentIdx )
+{
+  startFrame = 1;
+  endFrame = 1;
+  gtStartFrame = 1;
+  ratioSucc = 0;
+}
+
+TrackerTRETest::~TrackerTRETest()
+{
+
+}
+
+string TrackerTRETest::getRatioSucc() const
+{
+  stringstream ratio;
+  ratio << ratioSucc;
+  return ratio.str();
+}
+
+std::vector<std::string> TrackerTRETest::splitString( std::string s, std::string delimiter )
+{
+  std::vector<string> token;
+  size_t pos = 0;
+  while ( ( pos = s.find( delimiter ) ) != std::string::npos )
+  {
+    token.push_back( s.substr( 0, pos ) );
+    s.erase( 0, pos + delimiter.length() );
+  }
+  token.push_back( s );
+  return token;
+}
+
+float TrackerTRETest::calcDistance( Rect a, Rect b )
+{
+  Point2f p_a( a.x + a.width / 2, a.y + a.height / 2 );
+  Point2f p_b( b.x + b.width / 2, b.y + b.height / 2 );
+  return sqrt( pow( p_a.x - p_b.x, 2 ) + pow( p_a.y - p_b.y, 2 ) );
+}
+
+float TrackerTRETest::calcOverlap( Rect a, Rect b )
+{
+  float aArea = a.width * a.height;
+  float bArea = b.width * b.height;
+
+  if( aArea < bArea )
+  {
+    a.x -= ( b.width - a.width ) / 2;
+    a.y -= ( b.height - a.height ) / 2;
+    a.width = b.width;
+    a.height = b.height;
+  }
+  else
+  {
+    b.x -= ( a.width - b.width ) / 2;
+    b.y -= ( a.height - b.height ) / 2;
+    b.width = a.width;
+    b.height = a.height;
+  }
+
+  Rect rectIntersection = a & b;
+  Rect rectUnion = a | b;
+  float iArea = rectIntersection.width * rectIntersection.height;
+  float uArea = rectUnion.width * rectUnion.height;
+  float overlap = iArea / uArea;
+  return overlap;
+}
+
+void TrackerTRETest::distanceTest()
+{
+  Mat frame;
+  bool initialized = false;
+
+  int fc = ( startFrame - gtStartFrame );
+
+  Rect currentBB = bbs.at( fc );
+  float sumDistance = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  int frameTotal = 0;
+  int frameTotalSucc = 0;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+  for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+  {
+    c >> frame;
+    if( frame.empty() )
+    {
+      break;
+    }
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+
+    float curDistance = calcDistance( currentBB, bbs.at( fc ) );
+    if( curDistance <= threshold )
+      frameTotalSucc++;
+    sumDistance += curDistance;
+
+    fc++;
+    frameTotal++;
+  }
+
+  float distance = sumDistance / ( fc - ( startFrame - gtStartFrame ) );
+  ratioSucc = (float) frameTotalSucc / (float) frameTotal;
+
+  if( distance > threshold )
+  {
+    FAIL()<< "Incorrect distance: curr = " << distance << ", min = " << threshold << endl;
+    return;
+  }
+
+}
+
+void TrackerTRETest::overlapTest()
+{
+  Mat frame;
+  bool initialized = false;
+
+  int fc = ( startFrame - gtStartFrame );
+  Rect currentBB = bbs.at( fc );
+  float sumOverlap = 0;
+  string folder = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+
+  int frameTotal = 0;
+  int frameTotalSucc = 0;
+
+  VideoCapture c;
+  c.open( cvtest::TS::ptr()->get_data_path() + "/" + TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" );
+  c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+  for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+  {
+    c >> frame;
+    if( frame.empty() )
+    {
+      break;
+    }
+
+    if( !initialized )
+    {
+      if( !tracker->init( frame, currentBB ) )
+      {
+        FAIL()<< "Could not initialize tracker" << endl;
+        return;
+      }
+      initialized = true;
+    }
+    else if( initialized )
+    {
+      if( frameCounter >= (int) bbs.size() )
+      break;
+      tracker->update( frame, currentBB );
+    }
+    float curOverlap = calcOverlap( currentBB, bbs.at( fc ) );
+    if( curOverlap >= threshold )
+      frameTotalSucc++;
+
+    sumOverlap += curOverlap;
+    frameTotal++;
+    fc++;
+  }
+
+  float overlap = sumOverlap / ( fc - ( startFrame - gtStartFrame ) );
+  ratioSucc = (float) frameTotalSucc / (float) frameTotal;
+
+  if( overlap < threshold )
+  {
+    FAIL()<< "Incorrect overlap: curr = " << overlap << ", min = " << threshold << endl;
+    return;
+  }
+}
+
+void TrackerTRETest::checkDataTest()
+{
+
+  FileStorage fs;
+  fs.open( cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + video + ".yml", FileStorage::READ );
+  fs["start"] >> startFrame;
+  fs["prefix"] >> prefix;
+  fs["suffix"] >> suffix;
+  fs.release();
+
+  string gtFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/gt.txt";
+  ifstream gt;
+  //open the ground truth
+  gt.open( gtFile.c_str() );
+  if( !gt.is_open() )
+  {
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  }
+  string line;
+  int bbCounter = 0;
+  while ( getline( gt, line ) )
+  {
+    bbCounter++;
+  }
+  gt.close();
+
+  int seqLength = bbCounter;
+  for ( int i = startFrame; i < seqLength; i++ )
+  {
+    validSequence.push_back( i );
+  }
+
+  //exclude from the images sequence, the frames where the target is occluded or out of view
+  string omitFile = cvtest::TS::ptr()->get_data_path() + TRACKING_DIR + "/" + video + "/" + FOLDER_OMIT_INIT + "/" + video + ".txt";
+  ifstream omit;
+  omit.open( omitFile.c_str() );
+  if( omit.is_open() )
+  {
+    string omitLine;
+    while ( getline( omit, omitLine ) )
+    {
+      vector<string> tokens = splitString( omitLine, " " );
+      int s_start = atoi( tokens.at( 0 ).c_str() );
+      int s_end = atoi( tokens.at( 1 ).c_str() );
+      for ( int k = s_start; k <= s_end; k++ )
+      {
+        std::vector<int>::iterator position = std::find( validSequence.begin(), validSequence.end(), k );
+        if( position != validSequence.end() )
+          validSequence.erase( position );
+      }
+    }
+  }
+  omit.close();
+  gtStartFrame = startFrame;
+  //compute the start and the and for each segment
+  int segmentLength = sizeof ( SEGMENTS)/sizeof(int);
+  int numFrame = validSequence.size() / segmentLength;
+  startFrame += ( segmentIdx - 1 ) * numFrame;
+  endFrame = startFrame + numFrame;
+
+  ifstream gt2;
+  //open the ground truth
+  gt2.open( gtFile.c_str() );
+  if( !gt2.is_open() )
+  {
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  }
+  string line2;
+  int bbCounter2 = 0;
+  while ( getline( gt2, line2 ) )
+  {
+    vector<string> tokens = splitString( line2, "," );
+    Rect bb( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) );
+    if( tokens.size() != 4 )
+    {
+      FAIL()<< "Incorrect ground truth file";
+    }
+
+    bbs.push_back( bb );
+    bbCounter2++;
+  }
+  gt2.close();
+
+  if( segmentIdx == ( sizeof ( SEGMENTS)/sizeof(int) ) )
+  endFrame = bbs.size();
+
+}
+
+void TrackerTRETest::run()
+{
+  srand( 1 );
+  SCOPED_TRACE( "A" );
+
+  if( !tracker )
+  {
+    FAIL()<< "Error in the instantiation of the tracker" << endl;
+    return;
+  }
+
+  checkDataTest();
+
+  //check for failure
+  if( ::testing::Test::HasFatalFailure() )
+    return;
+
+  if( testType == DISTANCE )
+  {
+    distanceTest();
+  }
+  else if( testType == OVERLAP )
+  {
+    overlapTest();
+  }
+  else
+  {
+    FAIL()<< "Test type unknown" << endl;
+    return;
+  }
+
+}
+
+/****************************************************************************************\
+*                                Tests registrations                                     *
+ \****************************************************************************************/
+
+//[TESTDATA] [#SEGMENT] [LOCATION ERROR THRESHOLD]
+PARAM_TEST_CASE(TRE_Distance, string, int, float)
+{
+  int segment;
+  string dataset;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    segment = GET_PARAM(1);
+    threshold = GET_PARAM(2);
+  }
+};
+
+//[TESTDATA] [#SEGMENT] [OVERLAP THRESHOLD]
+PARAM_TEST_CASE(TRE_Overlap, string, int, float)
+{
+  int segment;
+  string dataset;
+  float threshold;
+  virtual void SetUp()
+  {
+    dataset = GET_PARAM(0);
+    segment = GET_PARAM(1);
+    threshold = GET_PARAM(2);
+  }
+};
+
+TEST_P(TRE_Distance, MIL)
+{
+  TrackerTRETest test( Tracker::create( "MIL" ), TrackerTRETest::DISTANCE, dataset, threshold, segment );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(TRE_Overlap, MIL)
+{
+  TrackerTRETest test( Tracker::create( "MIL" ), TrackerTRETest::OVERLAP, dataset, threshold, segment );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(TRE_Distance, Boosting)
+{
+  TrackerTRETest test( Tracker::create( "BOOSTING" ), TrackerTRETest::DISTANCE, dataset, threshold, segment );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+TEST_P(TRE_Overlap, Boosting)
+{
+  TrackerTRETest test( Tracker::create( "BOOSTING" ), TrackerTRETest::OVERLAP, dataset, threshold, segment );
+  test.run();
+  RecordProperty( "ratioSuccess", test.getRatioSucc() );
+}
+
+INSTANTIATE_TEST_CASE_P( Tracking, TRE_Distance, testing::Combine( TESTSET_NAMES, SEGMENTS, LOCATION_ERROR_THRESHOLD ) );
+
+INSTANTIATE_TEST_CASE_P( Tracking, TRE_Overlap, testing::Combine( TESTSET_NAMES, SEGMENTS, OVERLAP_THRESHOLD ) );
+
+/* End of file. */