noise control implementation started

noise control implementation ended

noise control errors fixed (part I)

noise control errors fixed (ended)

Perf Test for staticSaliencySpectralResidual and objectnessBING added

improvements and bugs fixed

saliency perf test added

matrices optimization

Where possible, CV_8U matrices have been used in place of CV_32F

new sorting algorithm implemented (first part)

new sorting algorithm implemented (second part)

new sorting algorithm implemented (finished)

modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp

@@ -234,15 +234,28 @@ private:
   bool templateOrdering();
   bool templateReplacement( const Mat& finalBFMask, const Mat& image );
 
+  // Decision threshold adaptation and Activity control function
+  bool activityControl(const Mat& current_noisePixelsMask);
+  bool decisionThresholdAdaptation();
+
   // changing structure
   std::vector<Ptr<Mat> > backgroundModel;// The vector represents the background template T0---TK of reference paper.
   // Matrices are two-channel matrix. In the first layer there are the B (background value)
   // for each pixel. In the second layer, there are the C (efficacy) value for each pixel
   Mat potentialBackground;// Two channel Matrix. For each pixel, in the first level there are the Ba value (potential background value)
                           // and in the secon level there are the Ca value, the counter for each potential value.
-  Mat epslonPixelsValue;  // epslon threshold
+  Mat epslonPixelsValue;// epslon threshold
+
+  Mat activityPixelsValue;// Activity level of each pixel
+
+  //vector<Mat> noisePixelMask; // We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution
+  Mat noisePixelMask;// We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution
+  //detection process,however, after the low resolution detection, it has become a
+  // background pixel. The matrix is  two-channel matrix. In the first layer there is the mask ( the identified noise-pixels are set to 1 while other pixels are 0)
+  // for each pixel. In the second layer, there is the value of activity level A for each pixel.
 
   //fixed parameter
+  bool activityControlFlag;
   bool neighborhoodCheck;
   int N_DS;// Number of template to be downsampled and used in lowResolutionDetection function
   CV_PROP_RW int imageWidth;// Width of input image
@@ -257,6 +270,13 @@ private:
             // long-term template, regardless of any subsequent background changes. A relatively large (eg gamma=3) will
             //restrain the generation of ghosts.
 
+  int Ainc;// Activity Incrementation;
+  int Bmax;// Upper-bound value for pixel activity
+  int Bth;// Max activity threshold
+  int Binc, Bdec;// Threshold for pixel-level decision threshold (epslon) adaptation
+  float deltaINC, deltaDEC;// Increment-decrement value for epslon adaptation
+  int epslonMIN, epslonMAX;// Range values for epslon threshold
+
 };
 
 /************************************ Specific Objectness Specialized Classes ************************************/
@@ -417,7 +437,7 @@ private:
   int _Clr;//
   static const char* _clrName[3];
 
-  // Names and paths to read model and to store results
+// Names and paths to read model and to store results
   std::string _modelName, _bbResDir, _trainingPath, _resultsDir;
 
   std::vector<int> _svmSzIdxs;// Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows
@@ -425,12 +445,12 @@ private:
   FilterTIG _tigF;// TIG filter
   Mat _svmReW1f;// Re-weight parameters learned at stage II.
 
-  // List of the rectangles' objectness value, in the same order as
-  // the  vector<Vec4i> objectnessBoundingBox returned by the algorithm (in computeSaliencyImpl function)
+// List of the rectangles' objectness value, in the same order as
+// the  vector<Vec4i> objectnessBoundingBox returned by the algorithm (in computeSaliencyImpl function)
   std::vector<float> objectnessValues;
 
 private:
-  // functions
+// functions
 
   inline static float LoG( float x, float y, float delta )
   {
@@ -438,17 +458,17 @@ private:
     return -1.0f / ( (float) ( CV_PI ) * pow( delta, 4 ) ) * ( 1 + d ) * exp( d );
   }  // Laplacian of Gaussian
 
-  // Read matrix from binary file
+// Read matrix from binary file
   static bool matRead( const std::string& filename, Mat& M );
 
   void setColorSpace( int clr = MAXBGR );
 
-  // Load trained model.
+// Load trained model.
   int loadTrainedModel( std::string modelName = "" );// Return -1, 0, or 1 if partial, none, or all loaded
 
-  // Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).
-  // The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().
-  // Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)
+// Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).
+// The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().
+// Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)
   void getObjBndBoxes( Mat &img3u, ValStructVec<float, Vec4i> &valBoxes, int numDetPerSize = 120 );
   void getObjBndBoxesForSingleImage( Mat img, ValStructVec<float, Vec4i> &boxes, int numDetPerSize );
 
@@ -460,7 +480,7 @@ private:
   void predictBBoxSI( Mat &mag3u, ValStructVec<float, Vec4i> &valBoxes, std::vector<int> &sz, int NUM_WIN_PSZ = 100, bool fast = true );
   void predictBBoxSII( ValStructVec<float, Vec4i> &valBoxes, const std::vector<int> &sz );
 
-  // Calculate the image gradient: center option as in VLFeat
+// Calculate the image gradient: center option as in VLFeat
   void gradientMag( Mat &imgBGR3u, Mat &mag1u );
 
   static void gradientRGB( Mat &bgr3u, Mat &mag1u );
@@ -479,7 +499,7 @@ private:
     return abs( u[0] - v[0] ) + abs( u[1] - v[1] ) + abs( u[2] - v[2] );
   }
 
-  //Non-maximal suppress
+//Non-maximal suppress
   static void nonMaxSup( Mat &matchCost1f, ValStructVec<float, Point> &matchCost, int NSS = 1, int maxPoint = 50, bool fast = true );
 
 };

modules/saliency/perf/perf_main.cpp

+#include "perf_precomp.hpp"
+
+CV_PERF_TEST_MAIN(saliency)

modules/saliency/perf/perf_motionSaliencyBinWangApr2014.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2014, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "perf_precomp.hpp"
+#include <fstream>
+#include <opencv2/videoio.hpp>
+
+//write sanity: ./bin/opencv_perf_saliency --perf_write_sanity=true
+//verify sanity: ./bin/opencv_perf_saliency
+
+using namespace std;
+using namespace cv;
+using namespace perf;
+
+#define TESTSET_NAMES \
+    "/cv/saliency/motion/blizzard.webm",\
+    "/cv/saliency/motion/pedestrian.webm" ,\
+    "/cv/saliency/motion/snowFall.webm"
+
+const string SALIENCY_DIR = "cv/saliency";
+
+typedef perf::TestBaseWithParam<std::string> sal;
+
+PERF_TEST_P(sal, motionSaliencyBinWangApr2014, testing::Values(TESTSET_NAMES))
+{
+  string filename = getDataPath(GetParam());
+  int startFrame=0;
+  Mat frame;
+  Mat saliencyMap;
+  int videoSize=0;
+
+  Ptr<saliency::Saliency> saliencyAlgorithm = saliency::Saliency::create( "BinWangApr2014" );
+
+  TEST_CYCLE_N(1)
+  {
+    VideoCapture c;
+    c.open( filename);
+    videoSize=c.get( CAP_PROP_FRAME_COUNT);
+    c.set( CAP_PROP_POS_FRAMES, startFrame );
+
+    for ( int frameCounter = 0; frameCounter < videoSize; frameCounter++ )
+    {
+      c >> frame;
+
+      if( frame.empty() )
+      {
+        break;
+      }
+
+      saliencyAlgorithm.dynamicCast<saliency::MotionSaliencyBinWangApr2014>()->setImagesize( frame.cols, frame.rows );
+      saliencyAlgorithm.dynamicCast<saliency::MotionSaliencyBinWangApr2014>()->init();
+
+      if( saliencyAlgorithm->computeSaliency( frame, saliencyMap ) )
+      {
+
+      }
+      else
+      {
+        FAIL()<< "***Error in the instantiation of the saliency algorithm...***\n" << endl;
+        return;
+      }
+
+    }
+  }
+
+  SANITY_CHECK(saliencyMap);
+
+}

modules/saliency/perf/perf_objectnessBING.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2014, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "perf_precomp.hpp"
+
+//write sanity: ./bin/opencv_perf_saliency --perf_write_sanity=true
+//verify sanity: ./bin/opencv_perf_saliency
+
+using namespace std;
+using namespace cv;
+using namespace perf;
+using std::tr1::make_tuple;
+using std::tr1::get;
+
+typedef perf::TestBaseWithParam<std::string> sal;
+
+#define BING_IMAGES \
+    "cv/saliency/objectness/000021.jpg", \
+"cv/saliency/objectness/000022.jpg"
+
+void getMatOfRects( const vector<Vec4i>& saliencyMap, Mat& bbs_mat )
+{
+  for ( size_t b = 0; b < saliencyMap.size(); b++ )
+  {
+    bbs_mat.at<int>( b, 0 ) = saliencyMap[b].val[0];
+    bbs_mat.at<int>( b, 1 ) = saliencyMap[b].val[1];
+    bbs_mat.at<int>( b, 2 ) = saliencyMap[b].val[2];
+    bbs_mat.at<int>( b, 3 ) = saliencyMap[b].val[3];
+  }
+}
+
+PERF_TEST_P(sal, objectnessBING, testing::Values(BING_IMAGES))
+{
+  string filename = getDataPath(GetParam());
+  cout<<endl<<endl<<"path "<<filename<<endl<<endl;
+  Mat image = imread(filename);
+  vector<Vec4i> saliencyMap;
+  String training_path = "/home/puja/src/opencv_contrib/modules/saliency/samples/ObjectnessTrainedModel";
+
+  if (image.empty())
+  FAIL() << "Unable to load source image " << filename;
+
+  Ptr<saliency::Saliency> saliencyAlgorithm = saliency::Saliency::create( "BING" );
+
+  TEST_CYCLE_N(1)
+  {
+    if( training_path.empty() )
+    {
+      FAIL() << "Path of trained files missing! " << endl;
+      return;
+    }
+
+    else
+    {
+      saliencyAlgorithm.dynamicCast<saliency::ObjectnessBING>()->setTrainingPath( training_path );
+      saliencyAlgorithm.dynamicCast<saliency::ObjectnessBING>()->setBBResDir( training_path + "/Results" );
+
+      if( saliencyAlgorithm->computeSaliency( image, saliencyMap ) )
+      {
+
+      }
+    }
+  }  //end CYCLE
+
+  //save the bounding boxes in a Mat
+   Mat bbs_mat( saliencyMap.size(), 4, CV_32F );
+   getMatOfRects( saliencyMap, bbs_mat );
+
+   SANITY_CHECK( bbs_mat);
+}

modules/saliency/perf/perf_precomp.hpp

+#ifdef __GNUC__
+#  pragma GCC diagnostic ignored "-Wmissing-declarations"
+#  if defined __clang__ || defined __APPLE__
+#    pragma GCC diagnostic ignored "-Wmissing-prototypes"
+#    pragma GCC diagnostic ignored "-Wextra"
+#  endif
+#endif
+
+#ifndef __OPENCV_SALIENCY_PRECOMP_HPP__
+#define __OPENCV_SALIENCY_PRECOMP_HPP__
+
+#include <opencv2/ts.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/saliency.hpp>
+#include <opencv2/highgui.hpp>
+
+#ifdef GTEST_CREATE_SHARED_LIBRARY
+#error no modules except ts should have GTEST_CREATE_SHARED_LIBRARY defined
+#endif
+
+#endif

modules/saliency/perf/perf_staticSaliencySpectralResidual.cpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+ //
+ //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+ //
+ //  By downloading, copying, installing or using the software you agree to this license.
+ //  If you do not agree to this license, do not download, install,
+ //  copy or use the software.
+ //
+ //
+ //                           License Agreement
+ //                For Open Source Computer Vision Library
+ //
+ // Copyright (C) 2014, OpenCV Foundation, all rights reserved.
+ // Third party copyrights are property of their respective owners.
+ //
+ // Redistribution and use in source and binary forms, with or without modification,
+ // are permitted provided that the following conditions are met:
+ //
+ //   * Redistribution's of source code must retain the above copyright notice,
+ //     this list of conditions and the following disclaimer.
+ //
+ //   * Redistribution's in binary form must reproduce the above copyright notice,
+ //     this list of conditions and the following disclaimer in the documentation
+ //     and/or other materials provided with the distribution.
+ //
+ //   * The name of the copyright holders may not be used to endorse or promote products
+ //     derived from this software without specific prior written permission.
+ //
+ // This software is provided by the copyright holders and contributors "as is" and
+ // any express or implied warranties, including, but not limited to, the implied
+ // warranties of merchantability and fitness for a particular purpose are disclaimed.
+ // In no event shall the Intel Corporation or contributors be liable for any direct,
+ // indirect, incidental, special, exemplary, or consequential damages
+ // (including, but not limited to, procurement of substitute goods or services;
+ // loss of use, data, or profits; or business interruption) however caused
+ // and on any theory of liability, whether in contract, strict liability,
+ // or tort (including negligence or otherwise) arising in any way out of
+ // the use of this software, even if advised of the possibility of such damage.
+ //
+ //M*/
+
+#include "perf_precomp.hpp"
+
+//write sanity: ./bin/opencv_perf_saliency --perf_write_sanity=true
+//verify sanity: ./bin/opencv_perf_saliency
+
+using namespace std;
+using namespace cv;
+using namespace perf;
+using std::tr1::make_tuple;
+using std::tr1::get;
+
+typedef perf::TestBaseWithParam<std::string> sal;
+
+#define STATIC_IMAGES \
+    "cv/saliency/static_saliency/8.jpg",\
+    "cv/saliency/static_saliency/39.jpg" ,\
+    "cv/saliency/static_saliency/41.jpg" ,\
+    "cv/saliency/static_saliency/62.jpg"
+
+PERF_TEST_P(sal, statiSaliencySpectralResidual, testing::Values(STATIC_IMAGES))
+{
+  string filename = getDataPath(GetParam());
+  Mat image = imread(filename);
+  Mat saliencyMap;
+
+  if (image.empty())
+  FAIL() << "Unable to load source image " << filename;
+
+  Ptr<saliency::Saliency> saliencyAlgorithm = saliency::Saliency::create( "SPECTRAL_RESIDUAL" );
+
+  TEST_CYCLE_N(1)
+  {
+
+    if( saliencyAlgorithm->computeSaliency( image, saliencyMap ) )
+    {
+
+    }
+    else
+    {
+      FAIL()<< "***Error in the instantiation of the saliency algorithm...***\n" << endl;
+      return;
+    }
+  } //end CYCLE
+
+  SANITY_CHECK(saliencyMap);
+}

modules/saliency/samples/computeSaliency.cpp

@@ -64,6 +64,7 @@ static void help()
 
 int main( int argc, char** argv )
 {
+
   CommandLineParser parser( argc, argv, keys );
 
   String saliency_algorithm = parser.get<String>( 0 );
@@ -175,12 +176,16 @@ int main( int argc, char** argv )
       {
 
         cap >> frame;
+        if( frame.empty() )
+        {
+          return 0;
+        }
         cvtColor( frame, frame, COLOR_BGR2GRAY );
 
         Mat saliencyMap;
         if( saliencyAlgorithm->computeSaliency( frame, saliencyMap ) )
         {
-          std::cout << "current frame motion saliency done" << std::endl;
+          //std::cout << "current frame motion saliency done" << std::endl;
         }
 
         imshow( "image", frame );