Added perf_test and test based on distance and overlap for tracking API

Added OPE tests tracking

- Reinitialized random number generator (TrackerMIL and TrackerBoosting)
- Removed unused meanSigmaPair

Changed sample tracker.cpp with a list of images instead the video file

Modified OPE tests tracking

- The overlap threshold varies from 0.0 to 1.0
- The location error threshold varies from 0 to 50

Changed name for the tests

Removed unused field

Enlarged first rect of 10% respect the ground truth

Added TRE test

TRE test for the temporal robustness evaluation

Added SRE Test (spatial robustness evaluation)

- Removed the enlargment of the first frame
- TRE corrected the computation of the distance and the overlap
- TrackerBoosting changed the search factor

Fixed last segment in test TRE

First stub for regression test

Added test for faceocc2

Added perf_test for mil and boosting trackers

Modified SANITY_CHECK with ERROR_RELATIVE

In xml generated added the ratio between of the correct bounding box (based on dynamic threshold)

OPE Test: works with video and not with sequence of images

TRE Test: works with video and not with sequence of images

SRE Test: works with video and not with sequence of images

Perf test: works with video and not with sequence of images

Removed unused file

sample tracker.cpp: works with video and not with sequence of images

modules/tracking/perf/perf_Tracker.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "perf_precomp.hpp"
+#include <fstream>
+
+using namespace std;
+using namespace cv;
+using namespace perf;
+using std::tr1::make_tuple;
+using std::tr1::get;
+
+//write sanity: ./bin/opencv_perf_tracking --perf_write_sanity=true --perf_min_samples=1
+//verify sanity: ./bin/opencv_perf_tracking --perf_min_samples=1
+
+#define TESTSET_NAMES testing::Values("david","dudek","faceocc2")
+//#define TESTSET_NAMES testing::internal::ValueArray1<string>("david")
+#define SEGMENTS testing::Values(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+
+const string TRACKING_DIR = "cv/tracking";
+const string FOLDER_IMG = "data";
+
+typedef perf::TestBaseWithParam<tr1::tuple<string, int> > tracking;
+
+std::vector<std::string> splitString( std::string s, std::string delimiter )
+{
+  std::vector<string> token;
+  size_t pos = 0;
+  while ( ( pos = s.find( delimiter ) ) != std::string::npos )
+  {
+    token.push_back( s.substr( 0, pos ) );
+    s.erase( 0, pos + delimiter.length() );
+  }
+  token.push_back( s );
+  return token;
+}
+
+void checkData( const string& datasetMeta, int& startFrame, string& prefix, string& suffix )
+{
+  //get informations on the current test data
+  FileStorage fs;
+  fs.open( datasetMeta, FileStorage::READ );
+  fs["start"] >> startFrame;
+  fs["prefix"] >> prefix;
+  fs["suffix"] >> suffix;
+  fs.release();
+}
+
+bool getGroundTruth( const string& gtFile, vector<Rect>& gtBBs )
+{
+  ifstream gt;
+  //open the ground truth
+  gt.open( gtFile.c_str() );
+  if( !gt.is_open() )
+  {
+    return false;
+  }
+  string line;
+  Rect currentBB;
+  while ( getline( gt, line ) )
+  {
+    vector<string> tokens = splitString( line, "," );
+
+    if( tokens.size() != 4 )
+    {
+      return false;
+    }
+
+    gtBBs.push_back(
+        Rect( atoi( tokens.at( 0 ).c_str() ), atoi( tokens.at( 1 ).c_str() ), atoi( tokens.at( 2 ).c_str() ), atoi( tokens.at( 3 ).c_str() ) ) );
+  }
+  return true;
+}
+
+void getSegment( int segmentId, int numSegments, int bbCounter, int& startFrame, int& endFrame )
+{
+  //compute the start and the and for each segment
+  int gtStartFrame = startFrame;
+  int numFrame = bbCounter / numSegments;
+  startFrame += ( segmentId - 1 ) * numFrame;
+  endFrame = startFrame + numFrame;
+
+  if( ( segmentId ) == numSegments )
+    endFrame = bbCounter + gtStartFrame - 1;
+}
+
+void getMatOfRects( const vector<Rect>& bbs, Mat& bbs_mat )
+{
+  for ( size_t b = 0; b < bbs.size(); b++ )
+  {
+    bbs_mat.at<float>( b, 0 ) = bbs[b].x;
+    bbs_mat.at<float>( b, 1 ) = bbs[b].y;
+    bbs_mat.at<float>( b, 2 ) = bbs[b].width;
+    bbs_mat.at<float>( b, 3 ) = bbs[b].height;
+  }
+}
+
+PERF_TEST_P(tracking, mil, testing::Combine(TESTSET_NAMES, SEGMENTS))
+{
+  string video = get<0>( GetParam() );
+  int segmentId = get<1>( GetParam() );
+
+  int startFrame;
+  string prefix;
+  string suffix;
+  string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" );
+  checkData( datasetMeta, startFrame, prefix, suffix );
+  int gtStartFrame = startFrame;
+
+  vector<Rect> gtBBs;
+  string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" );
+  if( !getGroundTruth( gtFile, gtBBs ) )
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  int bbCounter = gtBBs.size();
+
+  Mat frame;
+  bool initialized = false;
+  vector<Rect> bbs;
+
+  Ptr<Tracker> tracker = Tracker::create( "MIL" );
+  string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+  int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) );
+  int endFrame = 0;
+  getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame );
+
+  Rect currentBB = gtBBs[startFrame - gtStartFrame];
+
+  TEST_CYCLE_N(1)
+  {
+    VideoCapture c;
+    c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) );
+    c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+    for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+    {
+      c >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
+      if( !initialized )
+      {
+        if( !tracker->init( frame, currentBB ) )
+        {
+          FAIL()<< "Could not initialize tracker" << endl;
+          return;
+        }
+        initialized = true;
+      }
+      else if( initialized )
+      {
+        tracker->update( frame, currentBB );
+      }
+      bbs.push_back( currentBB );
+
+    }
+  }
+  //save the bounding boxes in a Mat
+  Mat bbs_mat( bbs.size(), 4, CV_32F );
+  getMatOfRects( bbs, bbs_mat );
+
+  SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE );
+
+}
+
+PERF_TEST_P(tracking, boosting, testing::Combine(TESTSET_NAMES, SEGMENTS))
+{
+  string video = get<0>( GetParam() );
+  int segmentId = get<1>( GetParam() );
+
+  int startFrame;
+  string prefix;
+  string suffix;
+  string datasetMeta = getDataPath( TRACKING_DIR + "/" + video + "/" + video + ".yml" );
+  checkData( datasetMeta, startFrame, prefix, suffix );
+  int gtStartFrame = startFrame;
+
+  vector<Rect> gtBBs;
+  string gtFile = getDataPath( TRACKING_DIR + "/" + video + "/gt.txt" );
+  if( !getGroundTruth( gtFile, gtBBs ) )
+    FAIL()<< "Ground truth file " << gtFile << " can not be read" << endl;
+  int bbCounter = gtBBs.size();
+
+  Mat frame;
+  bool initialized = false;
+  vector<Rect> bbs;
+
+  Ptr<Tracker> tracker = Tracker::create( "BOOSTING" );
+  string folder = TRACKING_DIR + "/" + video + "/" + FOLDER_IMG;
+  int numSegments = ( sizeof ( SEGMENTS)/sizeof(int) );
+  int endFrame = 0;
+  getSegment( segmentId, numSegments, bbCounter, startFrame, endFrame );
+
+  Rect currentBB = gtBBs[startFrame - gtStartFrame];
+
+  TEST_CYCLE_N(1)
+  {
+    VideoCapture c;
+    c.open( getDataPath( TRACKING_DIR + "/" + video + "/" + FOLDER_IMG + "/" + video + ".webm" ) );
+    c.set( CAP_PROP_POS_FRAMES, startFrame );
+    for ( int frameCounter = startFrame; frameCounter < endFrame; frameCounter++ )
+    {
+      c >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
+      if( !initialized )
+      {
+        if( !tracker->init( frame, currentBB ) )
+        {
+          FAIL()<< "Could not initialize tracker" << endl;
+          return;
+        }
+        initialized = true;
+      }
+      else if( initialized )
+      {
+        tracker->update( frame, currentBB );
+      }
+      bbs.push_back( currentBB );
+
+    }
+  }
+  //save the bounding boxes in a Mat
+  Mat bbs_mat( bbs.size(), 4, CV_32F );
+  getMatOfRects( bbs, bbs_mat );
+
+  SANITY_CHECK( bbs_mat, 15, ERROR_RELATIVE );
+
+}

modules/tracking/perf/perf_tracking.cpp

-/*M///////////////////////////////////////////////////////////////////////////////////////
- //
- //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
- //
- //  By downloading, copying, installing or using the software you agree to this license.
- //  If you do not agree to this license, do not download, install,
- //  copy or use the software.
- //
- //
- //                           License Agreement
- //                For Open Source Computer Vision Library
- //
- // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
- // Third party copyrights are property of their respective owners.
- //
- // Redistribution and use in source and binary forms, with or without modification,
- // are permitted provided that the following conditions are met:
- //
- //   * Redistribution's of source code must retain the above copyright notice,
- //     this list of conditions and the following disclaimer.
- //
- //   * Redistribution's in binary form must reproduce the above copyright notice,
- //     this list of conditions and the following disclaimer in the documentation
- //     and/or other materials provided with the distribution.
- //
- //   * The name of the copyright holders may not be used to endorse or promote products
- //     derived from this software without specific prior written permission.
- //
- // This software is provided by the copyright holders and contributors "as is" and
- // any express or implied warranties, including, but not limited to, the implied
- // warranties of merchantability and fitness for a particular purpose are disclaimed.
- // In no event shall the Intel Corporation or contributors be liable for any direct,
- // indirect, incidental, special, exemplary, or consequential damages
- // (including, but not limited to, procurement of substitute goods or services;
- // loss of use, data, or profits; or business interruption) however caused
- // and on any theory of liability, whether in contract, strict liability,
- // or tort (including negligence or otherwise) arising in any way out of
- // the use of this software, even if advised of the possibility of such damage.
- //
- //M*/
-
-#include "perf_precomp.hpp"
-
-using namespace std;
-using namespace cv;
-using namespace perf;

modules/tracking/samples/tracker.cpp

@@ -13,15 +13,17 @@ static bool selectObject = false;
 static bool startSelection = false;
 
 static const char* keys =
-{ "{@tracker_algorithm | | tracker algorithm }"
-    "{@video_name        | | video name        }" };
+{ "{@tracker_algorithm | | Tracker algorithm }"
+    "{@video_name      | | video name        }"
+    "{@start_frame     |1| Start frame       }" };
 
 static void help()
 {
   cout << "\nThis example shows the functionality of \"Long-term optical tracking API\""
        "-- pause video [p] and draw a bounding box around the target to start the tracker\n"
+       "Example of <video_name> is in opencv_extra/testdata/cv/tracking/\n"
        "Call:\n"
-       "./tracker <tracker_algorithm> <video_name>\n"
+       "./tracker <tracker_algorithm> <video_name> <start_frame>\n"
        << endl;
 
   cout << "\n\nHot keys: \n"
@@ -69,6 +71,7 @@ int main( int argc, char** argv )
 
   String tracker_algorithm = parser.get<String>( 0 );
   String video_name = parser.get<String>( 1 );
+  int start_frame = parser.get<int>( 2 );
 
   if( tracker_algorithm.empty() || video_name.empty() )
   {
@@ -79,6 +82,7 @@ int main( int argc, char** argv )
   //open the capture
   VideoCapture cap;
   cap.open( video_name );
+  cap.set( CAP_PROP_POS_FRAMES, start_frame );
 
   if( !cap.isOpened() )
   {
@@ -108,11 +112,19 @@ int main( int argc, char** argv )
   imshow( "Tracking API", image );
 
   bool initialized = false;
+  int frameCounter = 0;
+
   for ( ;; )
   {
     if( !paused )
     {
       cap >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
       frame.copyTo( image );
 
       if( !initialized && selectObject )
@@ -134,6 +146,7 @@ int main( int argc, char** argv )
         }
       }
       imshow( "Tracking API", image );
+      frameCounter++;
     }
 
     char c = (char) waitKey( 2 );

modules/tracking/src/trackerBoosting.cpp

@@ -56,7 +56,7 @@ TrackerBoosting::Params::Params()
 {
   numClassifiers = 100;
   samplerOverlap = 0.99f;
-  samplerSearchFactor = 2;
+  samplerSearchFactor = 1.8;
   iterationInit = 50;
   featureSetNumFeatures = ( numClassifiers * 10 ) + iterationInit;
 }
@@ -108,6 +108,7 @@ void TrackerBoosting::write( cv::FileStorage& fs ) const
 
 bool TrackerBoosting::initImpl( const Mat& image, const Rect& boundingBox )
 {
+  srand (1);
   //sampling
   Mat_<int> intImage;
   Mat_<double> intSqImage;

modules/tracking/src/trackerBoostingModel.hpp

@@ -99,7 +99,6 @@ class TrackerBoostingModel : public TrackerModel
  private:
 
   std::vector<Mat> currentSample;
-  std::vector<std::pair<float, float> > meanSigmaPair;
 
   int mode;
 };

modules/tracking/src/trackerMIL.cpp

@@ -124,6 +124,7 @@ void TrackerMIL::compute_integral( const Mat & img, Mat & ii_img )
 
 bool TrackerMIL::initImpl( const Mat& image, const Rect& boundingBox )
 {
+  srand (1);
   Mat intImage;
   compute_integral( image, intImage );
   TrackerSamplerCSC::Params CSCparameters;

modules/tracking/test/test_tracker.cpp

-/*M///////////////////////////////////////////////////////////////////////////////////////
- //
- //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
- //
- //  By downloading, copying, installing or using the software you agree to this license.
- //  If you do not agree to this license, do not download, install,
- //  copy or use the software.
- //
- //
- //                           License Agreement
- //                For Open Source Computer Vision Library
- //
- // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
- // Third party copyrights are property of their respective owners.
- //
- // Redistribution and use in source and binary forms, with or without modification,
- // are permitted provided that the following conditions are met:
- //
- //   * Redistribution's of source code must retain the above copyright notice,
- //     this list of conditions and the following disclaimer.
- //
- //   * Redistribution's in binary form must reproduce the above copyright notice,
- //     this list of conditions and the following disclaimer in the documentation
- //     and/or other materials provided with the distribution.
- //
- //   * The name of the copyright holders may not be used to endorse or promote products
- //     derived from this software without specific prior written permission.
- //
- // This software is provided by the copyright holders and contributors "as is" and
- // any express or implied warranties, including, but not limited to, the implied
- // warranties of merchantability and fitness for a particular purpose are disclaimed.
- // In no event shall the Intel Corporation or contributors be liable for any direct,
- // indirect, incidental, special, exemplary, or consequential damages
- // (including, but not limited to, procurement of substitute goods or services;
- // loss of use, data, or profits; or business interruption) however caused
- // and on any theory of liability, whether in contract, strict liability,
- // or tort (including negligence or otherwise) arising in any way out of
- // the use of this software, even if advised of the possibility of such damage.
- //
- //M*/
-
-#include "test_precomp.hpp"
-#include "opencv2/tracking.hpp"
-
-using namespace cv;
-using namespace std;
-
-class CV_TrackerBaseTest : public cvtest::BaseTest
-{
- public:
-  CV_TrackerBaseTest();
-  virtual ~CV_TrackerBaseTest();
-
-};
-
-CV_TrackerBaseTest::CV_TrackerBaseTest()
-{
-
-}
-
-CV_TrackerBaseTest::~CV_TrackerBaseTest()
-{
-
-}
-
-/************************************ TrackerMIL ************************************/
-
-class CV_TrackerMILTest : public CV_TrackerBaseTest
-{
- public:
-  CV_TrackerMILTest();
-  ~CV_TrackerMILTest();
-
- protected:
-  void run( int );
-};
-
-CV_TrackerMILTest::CV_TrackerMILTest()
-{
-}
-
-CV_TrackerMILTest::~CV_TrackerMILTest()
-{
-}
-
-void CV_TrackerMILTest::run( int )
-{
-  ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC );
-  ts->printf( cvtest::TS::LOG, "CV_TrackerMILTest to be implemented" );
-}
-
-TEST(DISABLED_Tracking_TrackerMIL, accuracy)
-{
-  CV_TrackerMILTest test;
-  test.safe_run();
-}
-
-/************************************ TrackerBoosting ************************************/
-
-class CV_TrackerBoostingTest : public CV_TrackerBaseTest
-{
- public:
-  CV_TrackerBoostingTest();
-  ~CV_TrackerBoostingTest();
-
- protected:
-  void run( int );
-};
-
-CV_TrackerBoostingTest::CV_TrackerBoostingTest()
-{
-}
-
-CV_TrackerBoostingTest::~CV_TrackerBoostingTest()
-{
-}
-
-void CV_TrackerBoostingTest::run( int )
-{
-  ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC );
-  ts->printf( cvtest::TS::LOG, "CV_TrackerBoostingTest to be implemented" );
-}
-
-TEST(DISABLED_Tracking_TrackerBoosting, accuracy)
-{
-  CV_TrackerBoostingTest test;
-  test.safe_run();
-}
-
-/* End of file. */