motionSaliencyBinWangApr2014.cpp 24.4 KB
Newer Older
jaco's avatar
jaco committed
1 2 3 4 5 6 7 8 9 10 11 12
/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
jaco's avatar
jaco committed
13
 // Copyright (C) 2014, OpenCV Foundation, all rights reserved.
jaco's avatar
jaco committed
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

42
#include <limits>
jaco's avatar
jaco committed
43 44
#include "precomp.hpp"

45 46
#define thetaA_VAL 200
#define thetaL_VAL 250
47
#define epslonGeneric 20
48

jaco's avatar
jaco committed
49 50
namespace cv
{
jaco's avatar
jaco committed
51 52
namespace saliency
{
jaco's avatar
jaco committed
53

jaco's avatar
jaco committed
54
void MotionSaliencyBinWangApr2014::setImagesize( int W, int H )
jaco's avatar
jaco committed
55
{
jaco's avatar
jaco committed
56 57
  imageWidth = W;
  imageHeight = H;
jaco's avatar
jaco committed
58 59 60 61
}

MotionSaliencyBinWangApr2014::MotionSaliencyBinWangApr2014()
{
62
  N_DS = 2;  // Number of template to be downsampled and used in lowResolutionDetection function
jaco's avatar
jaco committed
63
  K = 3;  // Number of background model template
jaco's avatar
jaco committed
64
  N = 4;   // NxN is the size of the block for downsampling in the lowlowResolutionDetection
jaco's avatar
jaco committed
65
  alpha = (float) 0.01;  // Learning rate
66 67
  L0 = 1000;  // Upper-bound values for C0 (efficacy of the first template (matrices) of backgroundModel
  L1 = 800;  // Upper-bound values for C1 (efficacy of the second template (matrices) of backgroundModel
68 69
  thetaL = thetaL_VAL;  // T0, T1 swap threshold
  thetaA = thetaA_VAL;
jaco's avatar
jaco committed
70
  gamma = 3;
jaco's avatar
jaco committed
71
  neighborhoodCheck = true;
jaco's avatar
jaco committed
72

73 74 75 76 77 78 79 80 81 82
  Ainc = 6;  // Activity Incrementation;
  Bmax = 80;  // Upper-bound value for pixel activity
  Bth = 20;  //70;  // Max activity threshold
  Binc = 15;  //50;
  Bdec = 5;  //20;  // Threshold for pixel-level decision threshold (epslon) adaptation
  deltaINC = 20;
  deltaDEC = 0.125;  // Increment-decrement value for epslon adaptation
  epslonMIN = 18;
  epslonMAX = 80;

jaco's avatar
jaco committed
83 84 85
  className = "BinWangApr2014";
}

86 87
bool MotionSaliencyBinWangApr2014::init()
{
88
  activityControlFlag = false;
jaco's avatar
jaco committed
89
  Size imgSize( imageWidth, imageHeight );
90 91
  epslonPixelsValue = Mat( imgSize.height, imgSize.width, CV_32F, Scalar( epslonGeneric ) );
  potentialBackground = Mat( imgSize.height, imgSize.width, CV_8UC2, Scalar( 0, 0 ) );
92
  backgroundModel.resize( K + 1 );
93

94
  for ( int i = 0; i < K + 1; i++ )
jaco's avatar
jaco committed
95
  {
96
    Mat* tmpm = new Mat;
jaco's avatar
jaco committed
97
    tmpm->create( imgSize.height, imgSize.width, CV_32FC2 );
98
    tmpm->setTo( Scalar( std::numeric_limits<float>::quiet_NaN(), 0 ) );
99 100
    Ptr<Mat> tmp = Ptr<Mat>( tmpm );
    backgroundModel[i] = tmp;
jaco's avatar
jaco committed
101
  }
102

103 104 105 106 107
  noisePixelMask.create( imgSize.height, imgSize.width, CV_8U );
  noisePixelMask.setTo( Scalar( 0 ) );
  activityPixelsValue.create( imgSize.height, imgSize.width, CV_8U );
  activityPixelsValue.setTo( Scalar( 0 ) );

108 109 110 111
  return true;

}

jaco's avatar
jaco committed
112 113 114 115 116 117
MotionSaliencyBinWangApr2014::~MotionSaliencyBinWangApr2014()
{

}

// classification (and adaptation) functions
jaco's avatar
jaco committed
118
bool MotionSaliencyBinWangApr2014::fullResolutionDetection( const Mat& image2, Mat& highResBFMask )
jaco's avatar
jaco committed
119
{
jaco's avatar
jaco committed
120
  Mat image = image2.clone();
121

122
  uchar currentPixelValue;
123
  float currentEpslonValue;
124
  bool backgFlag = false;
125 126

  // Initially, all pixels are considered as foreground and then we evaluate with the background model
127
  highResBFMask.create( image.rows, image.cols, CV_8U );
128 129
  highResBFMask.setTo( 1 );

jaco's avatar
jaco committed
130 131
  uchar* pImage;
  float* pEpslon;
132
  uchar* pMask;
jaco's avatar
jaco committed
133

134
  // Scan all pixels of image
jaco's avatar
jaco committed
135
  for ( int i = 0; i < image.rows; i++ )
136
  {
137

jaco's avatar
jaco committed
138 139
    pImage = image.ptr<uchar>( i );
    pEpslon = epslonPixelsValue.ptr<float>( i );
140
    pMask = highResBFMask.ptr<uchar>( i );
jaco's avatar
jaco committed
141
    for ( int j = 0; j < image.cols; j++ )
142
    {
143 144 145 146
      /*    Pixels with activity greater than Bth are eliminated from the detection result. In this way,
       continuously blinking noise-pixels will be eliminated from the detection results,
       preventing the generation of false positives.*/
      if( activityPixelsValue.at<uchar>( i, j ) < Bth )
147
      {
148 149 150
        backgFlag = false;
        currentPixelValue = pImage[j];
        currentEpslonValue = pEpslon[j];
151

152 153 154
        int counter = 0;
        for ( size_t z = 0; z < backgroundModel.size(); z++ )
        {
155

156 157 158 159
          counter += (int) backgroundModel[z]->ptr<Vec2f>( i )[j][1];
          if( counter != 0 )
            break;
        }
160

161
        if( counter != 0 )  //if at least the first template is activated / initialized
162
        {
jaco's avatar
jaco committed
163

164 165
          // scan background model vector
          for ( size_t z = 0; z < backgroundModel.size(); z++ )
166
          {
167 168 169 170 171 172 173
            float* currentB;
            float* currentC;
            currentB = & ( backgroundModel[z]->ptr<Vec2f>( i )[j][0] );
            currentC = & ( backgroundModel[z]->ptr<Vec2f>( i )[j][1] );

            //continue;
            if( ( *currentC ) > 0 )  //The current template is active
jaco's avatar
jaco committed
174
            {
175 176
              // If there is a match with a current background template
              if( abs( currentPixelValue - ( *currentB ) ) < currentEpslonValue && !backgFlag )
jaco's avatar
jaco committed
177
              {
178 179 180 181 182 183 184 185 186 187 188 189 190
                // The correspondence pixel in the  BF mask is set as background ( 0 value)
                pMask[j] = 0;
                if( ( *currentC < L0 && z == 0 ) || ( *currentC < L1 && z == 1 ) || ( z > 1 ) )
                {
                  *currentC += 1;  // increment the efficacy of this template
                }

                *currentB = ( ( 1 - alpha ) * ( *currentB ) ) + ( alpha * currentPixelValue );  // Update the template value
                backgFlag = true;
              }
              else
              {
                *currentC -= 1;  // decrement the efficacy of this template
191
              }
jaco's avatar
jaco committed
192 193 194

            }

195
          }  // end "for" cicle of template vector
196

197 198 199 200 201
        }
        else
        {
          pMask[j] = 1;  //if the model of the current pixel is not yet initialized, we mark the pixels as foreground
        }
jaco's avatar
jaco committed
202 203 204
      }
      else
      {
205
        pMask[j] = 0;
jaco's avatar
jaco committed
206
      }
jaco's avatar
jaco committed
207

208
    }
209
  }  // end "for" cicle of all image's pixels
210

jaco's avatar
jaco committed
211 212
  return true;
}
jaco's avatar
jaco committed
213 214

bool MotionSaliencyBinWangApr2014::lowResolutionDetection( const Mat& image, Mat& lowResBFMask )
jaco's avatar
jaco committed
215
{
jaco's avatar
jaco committed
216
  std::vector<Mat> mv;
217
  split( *backgroundModel[0], mv );
218

jaco's avatar
jaco committed
219
  //if at least the first template is activated / initialized for all pixels
220
  if( countNonZero( mv[1] ) > ( mv[1].cols * mv[1].rows ) / 2 )
jaco's avatar
jaco committed
221 222 223 224 225
  {
    float currentPixelValue;
    float currentEpslonValue;
    float currentB;
    float currentC;
jaco's avatar
jaco committed
226

jaco's avatar
jaco committed
227
    // Create a mask to select ROI in the original Image and Backgound model and at the same time compute the mean
jaco's avatar
jaco committed
228

jaco's avatar
jaco committed
229 230 231 232
    Rect roi( Point( 0, 0 ), Size( N, N ) );
    Scalar imageROImean;
    Scalar backGModelROImean;
    Mat currentModel;
jaco's avatar
jaco committed
233

jaco's avatar
jaco committed
234
    // Initially, all pixels are considered as foreground and then we evaluate with the background model
235
    lowResBFMask.create( image.rows, image.cols, CV_8U );
jaco's avatar
jaco committed
236
    lowResBFMask.setTo( 1 );
jaco's avatar
jaco committed
237

jaco's avatar
jaco committed
238
    // Scan all the ROI of original matrices
jaco's avatar
jaco committed
239
    for ( int i = 0; i < (int)ceil( (float) image.rows / N ); i++ )
jaco's avatar
jaco committed
240
    {
241 242 243 244 245 246
      if( ( roi.y + ( N - 1 ) ) <= ( image.rows - 1 ) )
      {
        // Reset original ROI dimension
        roi = Rect( Point( roi.x, roi.y ), Size( N, N ) );
      }

jaco's avatar
jaco committed
247
      for ( int j = 0; j < (int)ceil( (float) image.cols / N ); j++ )
jaco's avatar
jaco committed
248
      {
249 250 251 252
        /* Pixels with activity greater than Bth are eliminated from the detection result. In this way,
         continuously blinking noise-pixels will be eliminated from the detection results,
         preventing the generation of false positives.*/
        if( activityPixelsValue.at<uchar>( i, j ) < Bth )
253
        {
jaco's avatar
jaco committed
254

255 256 257 258 259 260 261 262
          // Compute the mean of image's block and epslonMatrix's block based on ROI
          Mat roiImage = image( roi );
          Mat roiEpslon = epslonPixelsValue( roi );
          currentPixelValue = (float) mean( roiImage ).val[0];
          currentEpslonValue = (float) mean( roiEpslon ).val[0];

          // scan background model vector
          for ( int z = 0; z < N_DS; z++ )
263
          {
264 265 266 267 268 269 270
            // Select the current template 2 channel matrix, select ROI and compute the mean for each channel separately
            Mat roiTemplate = ( * ( backgroundModel[z] ) )( roi );
            Scalar templateMean = mean( roiTemplate );
            currentB = (float) templateMean[0];
            currentC = (float) templateMean[1];

            if( ( currentC ) > 0 )  //The current template is active
jaco's avatar
jaco committed
271
            {
272 273 274 275 276 277 278
              // If there is a match with a current background template
              if( abs( currentPixelValue - ( currentB ) ) < currentEpslonValue )
              {
                // The correspondence pixel in the  BF mask is set as background ( 0 value)
                rectangle( lowResBFMask, roi, Scalar( 0 ), FILLED );
                break;
              }
jaco's avatar
jaco committed
279
            }
280
          }
281 282 283 284 285 286 287 288 289 290
          // Shift the ROI from left to right follow the block dimension
          roi = roi + Point( N, 0 );
          if( ( roi.x + ( roi.width - 1 ) ) > ( image.cols - 1 ) && ( roi.y + ( N - 1 ) ) <= ( image.rows - 1 ) )
          {
            roi = Rect( Point( roi.x, roi.y ), Size( abs( ( image.cols - 1 ) - roi.x ) + 1, N ) );
          }
          else if( ( roi.x + ( roi.width - 1 ) ) > ( image.cols - 1 ) && ( roi.y + ( N - 1 ) ) > ( image.rows - 1 ) )
          {
            roi = Rect( Point( roi.x, roi.y ), Size( abs( ( image.cols - 1 ) - roi.x ) + 1, abs( ( image.rows - 1 ) - roi.y ) + 1 ) );
          }
291
        }
292
        else
293
        {
294 295
          // The correspondence pixel in the  BF mask is set as background ( 0 value)
          rectangle( lowResBFMask, roi, Scalar( 0 ), FILLED );
296
        }
jaco's avatar
jaco committed
297
      }
jaco's avatar
jaco committed
298 299 300
      //Shift the ROI from up to down follow the block dimension, also bringing it back to beginning of row
      roi.x = 0;
      roi.y += N;
301 302
      if( ( roi.y + ( roi.height - 1 ) ) > ( image.rows - 1 ) )
      {
303
        roi = Rect( Point( roi.x, roi.y ), Size( N, abs( ( image.rows - 1 ) - roi.y ) + 1 ) );
304 305
      }

jaco's avatar
jaco committed
306
    }
jaco's avatar
jaco committed
307 308 309 310
    return true;
  }
  else
  {
311
    lowResBFMask.create( image.rows, image.cols, CV_8U );
312
    lowResBFMask.setTo( 1 );
jaco's avatar
jaco committed
313
    return false;
jaco's avatar
jaco committed
314 315
  }

jaco's avatar
jaco committed
316
}
317

jaco's avatar
jaco committed
318
bool inline pairCompare( std::pair<float, float> t, std::pair<float, float> t_plusOne )
jaco's avatar
jaco committed
319 320 321 322 323 324
{

  return ( t.second > t_plusOne.second );

}

jaco's avatar
jaco committed
325 326
bool MotionSaliencyBinWangApr2014::templateOrdering()
{
327

328 329
  Mat dstMask, tempMat, dstMask2, dstMask3;
  Mat convertMat1, convertMat2;
jaco's avatar
jaco committed
330
  int backGroundModelSize = (int)backgroundModel.size();
331 332 333 334 335 336 337

  std::vector<std::vector<Mat> > channelSplit( backGroundModelSize );
  for ( int i = 0; i < backGroundModelSize; i++ )
  {
    split( *backgroundModel[i], channelSplit[i] );

  }
jaco's avatar
jaco committed
338

339 340
  //Bubble sort : Template T1 - Tk
  for ( int i = 1; i < backGroundModelSize - 1; i++ )
jaco's avatar
jaco committed
341
  {
342 343
    // compare and order the i-th template with the others
    for ( int j = i + 1; j < backGroundModelSize; j++ )
jaco's avatar
jaco committed
344 345
    {

346
      compare( channelSplit[j][1], channelSplit[i][1], dstMask, CMP_GT );
jaco's avatar
jaco committed
347

348 349 350
      channelSplit[i][0].copyTo( tempMat );
      channelSplit[j][0].copyTo( channelSplit[i][0], dstMask );
      tempMat.copyTo( channelSplit[j][0], dstMask );
jaco's avatar
jaco committed
351

352 353 354 355 356
      channelSplit[i][1].copyTo( tempMat );
      channelSplit[j][1].copyTo( channelSplit[i][1], dstMask );
      tempMat.copyTo( channelSplit[j][1], dstMask );
    }
  }
jaco's avatar
jaco committed
357

358 359
  // SORT Template T0 and T1
  Mat M_deltaL( backgroundModel[0]->rows, backgroundModel[0]->cols, CV_32F, Scalar( thetaL ) );
jaco's avatar
jaco committed
360

361 362
  compare( channelSplit[1][1], M_deltaL, dstMask2, CMP_GT );
  compare( M_deltaL, channelSplit[0][1], dstMask3, CMP_GT );
jaco's avatar
jaco committed
363

364 365
  threshold( dstMask2, dstMask2, 0, 1, THRESH_BINARY );
  threshold( dstMask3, dstMask3, 0, 1, THRESH_BINARY );
jaco's avatar
jaco committed
366

367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
  bitwise_and( dstMask2, dstMask3, dstMask );

  //copy correct B element of T1 inside T0 and swap
  channelSplit[0][0].copyTo( tempMat );
  channelSplit[1][0].copyTo( channelSplit[0][0], dstMask );
  tempMat.copyTo( channelSplit[1][0], dstMask );

  //copy correct C element of T0 inside T1
  channelSplit[0][1].copyTo( channelSplit[1][1], dstMask );

  //set new C0 values as gamma * thetaL
  M_deltaL.mul( gamma );
  M_deltaL.copyTo( channelSplit[0][1], dstMask );

  for ( int i = 0; i < backGroundModelSize; i++ )
  {
    merge( channelSplit[i], *backgroundModel[i] );
jaco's avatar
jaco committed
384
  }
jaco's avatar
jaco committed
385 386 387

  return true;
}
388

389
bool MotionSaliencyBinWangApr2014::templateReplacement( const Mat& finalBFMask, const Mat& image )
jaco's avatar
jaco committed
390
{
jaco's avatar
jaco committed
391
  std::vector<Mat> temp;
392
  split( *backgroundModel[0], temp );
jaco's avatar
jaco committed
393

394
//if at least the first template is activated / initialized for all pixels
395
  if( countNonZero( temp[1] ) <= ( temp[1].cols * temp[1].rows ) / 2 )
jaco's avatar
jaco committed
396
  {
397 398
    thetaA = 50;
    thetaL = 150;
399 400
    /*    thetaA = 5;
     thetaL = 15;*/
jaco's avatar
jaco committed
401 402 403 404 405
    neighborhoodCheck = false;

  }
  else
  {
406 407
    thetaA = thetaA_VAL;
    thetaL = thetaL_VAL;
408
    neighborhoodCheck = true;
jaco's avatar
jaco committed
409 410
  }

jaco's avatar
jaco committed
411
  int roiSize = 3;  // FIXED ROI SIZE, not change until you first appropriately adjust the following controls in the EVALUATION section!
412
  int countNonZeroElements = 0;
413
  std::vector<Mat> mv;
414
  Mat replicateCurrentBAMat( roiSize, roiSize, CV_8U );
jaco's avatar
jaco committed
415
  Mat backgroundModelROI( roiSize, roiSize, CV_32F );
416
  Mat diffResult( roiSize, roiSize, CV_8U );
jaco's avatar
jaco committed
417

418
// Scan all pixels of finalBFMask and all pixels of others models (the dimension are the same)
419 420
  const uchar* finalBFMaskP;
  Vec2b* pbgP;
421 422
  const uchar* imageP;
  float* epslonP;
423 424
  for ( int i = 0; i < finalBFMask.rows; i++ )
  {
425 426
    finalBFMaskP = finalBFMask.ptr<uchar>( i );
    pbgP = potentialBackground.ptr<Vec2b>( i );
427 428
    imageP = image.ptr<uchar>( i );
    epslonP = epslonPixelsValue.ptr<float>( i );
429 430
    for ( int j = 0; j < finalBFMask.cols; j++ )
    {
431
      /////////////////// MAINTENANCE of potentialBackground model ///////////////////
432
      if( finalBFMaskP[j] == 1 )  // i.e. the corresponding frame pixel has been market as foreground
433 434 435
      {
        /* For the pixels with CA= 0, if the current frame pixel has been classified as foreground, its value
         * will be loaded into BA and CA will be set to 1*/
436
        if( pbgP[j][1] == 0 )
437
        {
438
          pbgP[j][0] = imageP[j];
439
          pbgP[j][1] = 1;
440 441 442 443
        }

        /*the distance between this pixel value and BA is calculated, and if this distance is smaller than
         the decision threshold epslon, then CA is increased by 1, otherwise is decreased by 1*/
444
        else if( abs( (float) imageP[j] - pbgP[j][0] ) < epslonP[j] )
445
        {
446
          pbgP[j][1] += 1;
447 448 449
        }
        else
        {
450
          pbgP[j][1] -= 1;
451
        }
452 453
        /*}*/  /////////////////// END of potentialBackground model MAINTENANCE///////////////////
        /////////////////// EVALUATION of potentialBackground values ///////////////////
454
        if( pbgP[j][1] > thetaA )
455
        {
jaco's avatar
jaco committed
456
          if( neighborhoodCheck )
457
          {
jaco's avatar
jaco committed
458
            // replicate currentBA value
459
            replicateCurrentBAMat.setTo( pbgP[j][0] );
460

jaco's avatar
jaco committed
461
            for ( size_t z = 0; z < backgroundModel.size(); z++ )
462
            {
jaco's avatar
jaco committed
463 464 465
              // Neighborhood of current pixel in the current background model template.
              // The ROI is centered in the pixel coordinates

466
              if( i > 0 && j > 0 && i < ( backgroundModel[z]->rows - 1 ) && j < ( backgroundModel[z]->cols - 1 ) )
jaco's avatar
jaco committed
467
              {
468
                split( *backgroundModel[z], mv );
469
                backgroundModelROI = mv[0]( Rect( j - (int) floor((float) roiSize / 2 ), i - (int) floor((float) roiSize / 2 ), roiSize, roiSize ) );
jaco's avatar
jaco committed
470
              }
jaco's avatar
jaco committed
471
              else if( i == 0 && j == 0 )  // upper leftt
jaco's avatar
jaco committed
472
              {
473
                split( *backgroundModel[z], mv );
474
                backgroundModelROI = mv[0]( Rect( j, i, (int) ceil((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
475
              }
476
              else if( j == 0 && i > 0 && i < ( backgroundModel[z]->rows - 1 ) )  // middle left
jaco's avatar
jaco committed
477
              {
478
                split( *backgroundModel[z], mv );
479
                backgroundModelROI = mv[0]( Rect( j, i - (int) floor((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ), roiSize ) );
jaco's avatar
jaco committed
480
              }
481
              else if( i == ( backgroundModel[z]->rows - 1 ) && j == 0 )  //down left
jaco's avatar
jaco committed
482
              {
483
                split( *backgroundModel[z], mv );
484
                backgroundModelROI = mv[0]( Rect( j, i - (int) floor((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
485
              }
486
              else if( i == 0 && j > 0 && j < ( backgroundModel[z]->cols - 1 ) )  // upper - middle
jaco's avatar
jaco committed
487
              {
488
                split( *backgroundModel[z], mv );
489
                backgroundModelROI = mv[0]( Rect( ( j - (int) floor((float) roiSize / 2 ) ), i, roiSize, (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
490
              }
491
              else if( i == ( backgroundModel[z]->rows - 1 ) && j > 0 && j < ( backgroundModel[z]->cols - 1 ) )  //down middle
jaco's avatar
jaco committed
492
              {
493
                split( *backgroundModel[z], mv );
jaco's avatar
jaco committed
494
                backgroundModelROI = mv[0](
495
                    Rect( j - (int) floor((float) roiSize / 2 ), i - (int) floor((float) roiSize / 2 ), roiSize, (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
496
              }
497
              else if( i == 0 && j == ( backgroundModel[z]->cols - 1 ) )  // upper right
jaco's avatar
jaco committed
498
              {
499
                split( *backgroundModel[z], mv );
500
                backgroundModelROI = mv[0]( Rect( j - (int) floor((float) roiSize / 2 ), i, (int) ceil((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
501
              }
502
              else if( j == ( backgroundModel[z]->cols - 1 ) && i > 0 && i < ( backgroundModel[z]->rows - 1 ) )  // middle - right
jaco's avatar
jaco committed
503
              {
504
                split( *backgroundModel[z], mv );
jaco's avatar
jaco committed
505
                backgroundModelROI = mv[0](
506
                    Rect( j - (int) floor((float) roiSize / 2 ), i - (int) floor((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ), roiSize ) );
jaco's avatar
jaco committed
507
              }
508
              else if( i == ( backgroundModel[z]->rows - 1 ) && j == ( backgroundModel[z]->cols - 1 ) )  // down right
jaco's avatar
jaco committed
509
              {
510
                split( *backgroundModel[z], mv );
jaco's avatar
jaco committed
511
                backgroundModelROI = mv[0](
512
                    Rect( j - (int) floor((float) roiSize / 2 ), i - (int) floor((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ), (int) ceil((float) roiSize / 2 ) ) );
jaco's avatar
jaco committed
513 514 515 516 517
              }

              /* Check if the value of current pixel BA in potentialBackground model is already contained in at least one of its neighbors'
               * background model
               */
518 519
              resize( replicateCurrentBAMat, replicateCurrentBAMat, Size( backgroundModelROI.cols, backgroundModelROI.rows ), 0, 0, INTER_LINEAR_EXACT );
              resize( diffResult, diffResult, Size( backgroundModelROI.cols, backgroundModelROI.rows ), 0, 0, INTER_LINEAR_EXACT );
jaco's avatar
jaco committed
520

521 522
              backgroundModelROI.convertTo( backgroundModelROI, CV_8U );

jaco's avatar
jaco committed
523
              absdiff( replicateCurrentBAMat, backgroundModelROI, diffResult );
524
              threshold( diffResult, diffResult, epslonP[j], 255, THRESH_BINARY_INV );
jaco's avatar
jaco committed
525 526 527 528 529 530
              countNonZeroElements = countNonZero( diffResult );

              if( countNonZeroElements > 0 )
              {
                /////////////////// REPLACEMENT of backgroundModel template ///////////////////
                //replace TA with current TK
531 532 533
                backgroundModel[backgroundModel.size() - 1]->at<Vec2f>( i, j ) = potentialBackground.at<Vec2b>( i, j );
                potentialBackground.at<Vec2b>( i, j )[0] = 0;
                potentialBackground.at<Vec2b>( i, j )[1] = 0;
534

jaco's avatar
jaco committed
535 536 537 538 539 540
                break;
              }
            }  // end for backgroundModel size
          }
          else
          {
541 542 543
            backgroundModel[backgroundModel.size() - 1]->at<Vec2f>( i, j ) = potentialBackground.at<Vec2b>( i, j );
            potentialBackground.at<Vec2b>( i, j )[0] = 0;
            potentialBackground.at<Vec2b>( i, j )[1] = 0;
jaco's avatar
jaco committed
544
          }
545 546
        }  // close if of EVALUATION
      }  // end of  if( finalBFMask.at<uchar>( i, j ) == 1 )  // i.e. the corresponding frame pixel has been market as foreground
547

548 549
    }  // end of second for
  }  // end of first for
550

jaco's avatar
jaco committed
551 552 553
  return true;
}

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
bool MotionSaliencyBinWangApr2014::activityControl( const Mat& current_noisePixelsMask )
{
  Mat discordanceFramesNoise, not_current_noisePixelsMask;
  Mat nonZeroIndexes, not_discordanceFramesNoise;  //u_current_noisePixelsMask;

//current_noisePixelsMask.convertTo( u_current_noisePixelsMask, CV_8UC1 );

// Derive the discrepancy between noise in the frame n-1 and frame n
//threshold( u_current_noisePixelsMask, not_current_noisePixelsMask, 0.5, 1.0, THRESH_BINARY_INV );
  threshold( current_noisePixelsMask, not_current_noisePixelsMask, 0.5, 1.0, THRESH_BINARY_INV );
  bitwise_and( noisePixelMask, not_current_noisePixelsMask, discordanceFramesNoise );

// indices in which the pixel at frame n-1 was the noise (or not) and now no (or yes) (blinking pixels)
  findNonZero( discordanceFramesNoise, nonZeroIndexes );

  Vec2i temp;

// we increase the activity value of these pixels
  for ( int i = 0; i < nonZeroIndexes.rows; i++ )
  {
    //TODO check rows, cols inside at
    temp = nonZeroIndexes.at<Vec2i>( i );
    if( activityPixelsValue.at<uchar>( temp.val[1], temp.val[0] ) < Bmax )
    {
      activityPixelsValue.at<uchar>( temp.val[1], temp.val[0] ) += Ainc;
    }
  }

// decrement other pixels that have not changed (not blinking)
  threshold( discordanceFramesNoise, not_discordanceFramesNoise, 0.5, 1.0, THRESH_BINARY_INV );
  findNonZero( not_discordanceFramesNoise, nonZeroIndexes );

  Vec2i temp2;

  for ( int j = 0; j < nonZeroIndexes.rows; j++ )
  {
    temp2 = nonZeroIndexes.at<Vec2i>( j );
    if( activityPixelsValue.at<uchar>( temp2.val[1], temp2.val[0] ) > 0 )
    {
      activityPixelsValue.at<uchar>( temp2.val[1], temp2.val[0] ) -= 1;
    }
  }
// update the noisePixelsMask
  current_noisePixelsMask.copyTo( noisePixelMask );

  return true;
}

bool MotionSaliencyBinWangApr2014::decisionThresholdAdaptation()
jaco's avatar
jaco committed
603
{
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623

  for ( int i = 0; i < activityPixelsValue.rows; i++ )
  {
    for ( int j = 0; j < activityPixelsValue.cols; j++ )
    {
      if( activityPixelsValue.at<uchar>( i, j ) > Binc && ( epslonPixelsValue.at<float>( i, j ) + deltaINC ) < epslonMAX )
      {

        epslonPixelsValue.at<float>( i, j ) += deltaINC;
      }
      else if( activityPixelsValue.at<uchar>( i, j ) < Bdec && ( epslonPixelsValue.at<float>( i, j ) - deltaDEC ) > epslonMIN )
      {
        epslonPixelsValue.at<float>( i, j ) -= deltaDEC;
      }
    }
  }

  return true;
}

624
bool MotionSaliencyBinWangApr2014::computeSaliencyImpl( InputArray image, OutputArray saliencyMap )
625
{
626 627
  CV_Assert(image.channels() == 1);

628 629
  Mat highResBFMask, u_highResBFMask;
  Mat lowResBFMask, u_lowResBFMask;
630
  Mat not_lowResBFMask;
631
  Mat current_noisePixelsMask;
jaco's avatar
jaco committed
632

633 634
  fullResolutionDetection( image.getMat(), highResBFMask );
  lowResolutionDetection( image.getMat(), lowResBFMask );
jaco's avatar
jaco committed
635

636 637
// Compute the final background-foreground mask. One pixel is marked as foreground if and only if it is
// foreground in both masks (full and low)
638
  bitwise_and( highResBFMask, lowResBFMask, saliencyMap );
jaco's avatar
jaco committed
639

640 641 642 643 644 645 646 647 648 649 650
  if( activityControlFlag )
  {

// Detect the noise pixels (i.e. for a given pixel, fullRes(pixel) = foreground and lowRes(pixel)= background)
    threshold( lowResBFMask, not_lowResBFMask, 0.5, 1.0, THRESH_BINARY_INV );
    bitwise_and( highResBFMask, not_lowResBFMask, current_noisePixelsMask );

    activityControl( current_noisePixelsMask );
    decisionThresholdAdaptation();
  }

651
  templateOrdering();
652
  templateReplacement( saliencyMap.getMat(), image.getMat() );
653
  templateOrdering();
jaco's avatar
jaco committed
654

655
  activityControlFlag = true;
jaco's avatar
jaco committed
656 657 658
  return true;
}

jaco's avatar
jaco committed
659
}  // namespace saliency
jaco's avatar
jaco committed
660
}  // namespace cv