tldTracker.cpp

/*///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include "tldTracker.hpp"

namespace cv
{

	TrackerTLD::Params::Params(){}

	void TrackerTLD::Params::read(const cv::FileNode& /*fn*/){}

	void TrackerTLD::Params::write(cv::FileStorage& /*fs*/) const {}


Ptr<TrackerTLD> TrackerTLD::create(const TrackerTLD::Params &parameters)
{
    return Ptr<tld::TrackerTLDImpl>(new tld::TrackerTLDImpl(parameters));
}
Ptr<TrackerTLD> TrackerTLD::create()
{
    return Ptr<tld::TrackerTLDImpl>(new tld::TrackerTLDImpl());
}

namespace tld
{

TrackerTLDImpl::TrackerTLDImpl(const TrackerTLD::Params &parameters) :
    params( parameters )
{
  isInit = false;
  trackerProxy = Ptr<TrackerProxyImpl<TrackerMedianFlow, TrackerMedianFlow::Params> >
      (new TrackerProxyImpl<TrackerMedianFlow, TrackerMedianFlow::Params>());
}

void TrackerTLDImpl::read(const cv::FileNode& fn)
{
  params.read( fn );
}

void TrackerTLDImpl::write(cv::FileStorage& fs) const
{
  params.write( fs );
}

bool TrackerTLDImpl::initImpl(const Mat& image, const Rect2d& boundingBox)
{
    Mat image_gray;
    trackerProxy->init(image, boundingBox);
    if(image.channels() > 1)
    {
        cvtColor( image, image_gray, COLOR_BGR2GRAY );
    }
    else
    {
        image_gray = image.clone();
    }

    data = Ptr<Data>(new Data(boundingBox));
    double scale = data->getScale();
    Rect2d myBoundingBox = boundingBox;
    if( scale > 1.0 )
    {
        Mat image_proxy;
        resize(image_gray, image_proxy, Size(cvRound(image.cols * scale), cvRound(image.rows * scale)), 0, 0, DOWNSCALE_MODE);
        image_proxy.copyTo(image_gray);
        myBoundingBox.x *= scale;
        myBoundingBox.y *= scale;
        myBoundingBox.width *= scale;
        myBoundingBox.height *= scale;
    }
    model = Ptr<TrackerTLDModel>(new TrackerTLDModel(params, image_gray, myBoundingBox, data->getMinSize()));

    data->confident = false;
    data->failedLastTime = false;

    return true;
}

bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox)
{
    Mat image_gray, image_blurred, imageForDetector;
    if(image.channels() > 1)
    {
        cvtColor( image, image_gray, COLOR_BGR2GRAY );
    }
    else
    {
        image_gray = image.clone();
    }
    double scale = data->getScale();
    if( scale > 1.0 )
        resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, DOWNSCALE_MODE);
    else
        imageForDetector = image_gray;
    GaussianBlur(imageForDetector, image_blurred, GaussBlurKernelSize, 0.0);
    TrackerTLDModel* tldModel = ((TrackerTLDModel*)static_cast<TrackerModel*>(model));
    data->frameNum++;
    Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
    std::vector<TLDDetector::LabeledPatch> detectorResults;
    //best overlap around 92%
    std::vector<Rect2d> candidates;
    std::vector<double> candidatesRes;
    bool trackerNeedsReInit = false;
    bool DETECT_FLG = false;

    //run tracker
    Rect2d tmpCandid = boundingBox;
    if(!data->failedLastTime && trackerProxy->update(image, tmpCandid))
    {
        candidates.push_back(tmpCandid);
        resample(image_gray, tmpCandid, standardPatch);
        candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
    }
    else
        trackerNeedsReInit = true;

    //run detector
    tmpCandid = boundingBox;
#ifdef HAVE_OPENCL
    if (false)//ocl::useOpenCL())
        DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
    else
#endif
        DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());

    if(DETECT_FLG)
    {
        candidates.push_back(tmpCandid);
        resample(imageForDetector, tmpCandid, standardPatch);
        candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
    }

    std::vector<double>::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end());

    if( it == candidatesRes.end() ) //candidates are empty
    {
        data->confident = false;
        data->failedLastTime = true;
        return false;
    }
    else
    {
        boundingBox = candidates[it - candidatesRes.begin()];
        data->failedLastTime = false;
        if( trackerNeedsReInit || it != candidatesRes.begin() )
            trackerProxy->init(image, boundingBox);
    }

#if 1
    if( it != candidatesRes.end() )
        resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch);
#endif

    if( *it > CORE_THRESHOLD )
        data->confident = true;

    if( data->confident )
    {
        Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, params, data->getMinSize());
		Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, params);
        std::vector<Mat_<uchar> > examplesForModel, examplesForEnsemble;
        examplesForModel.reserve(100); examplesForEnsemble.reserve(100);
        int negRelabeled = 0;
        for( int i = 0; i < (int)detectorResults.size(); i++ )
        {
            bool expertResult;
            if( detectorResults[i].isObject )
            {
                expertResult = nExpert(detectorResults[i].rect);
                if( expertResult != detectorResults[i].isObject )
                    negRelabeled++;
            }
            else
            {
                expertResult = pExpert(detectorResults[i].rect);
            }

            detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult);
            detectorResults[i].isObject = expertResult;
        }
        tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults);
        pExpert.additionalExamples(examplesForModel, examplesForEnsemble);
#ifdef HAVE_OPENCL
        if (false)//ocl::useOpenCL())
            tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true);
        else
#endif
        tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true);
        examplesForModel.clear(); examplesForEnsemble.clear();
        nExpert.additionalExamples(examplesForModel, examplesForEnsemble);

#ifdef HAVE_OPENCL
        if (false)//ocl::useOpenCL())
            tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false);
        else
#endif
            tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false);
    }
    else
    {
#ifdef CLOSED_LOOP
        tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults);
#endif
    }

    return true;
}


int TrackerTLDImpl::Pexpert::additionalExamples(std::vector<Mat_<uchar> >& examplesForModel, std::vector<Mat_<uchar> >& examplesForEnsemble)
{
    examplesForModel.clear(); examplesForEnsemble.clear();
    examplesForModel.reserve(100); examplesForEnsemble.reserve(100);

    std::vector<Rect2d> closest, scanGrid;
    Mat scaledImg, blurredImg;

    double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)),
            scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP);

    TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid);
    getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest);

    for( size_t i = 0; i < closest.size(); i++ )
    {
        for( size_t j = 0; j < 10; j++ )
        {
            Point2f center;
            Size2f size;
            Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(initSize_);
            center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01)));
            center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01)));
            size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01));
            size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01));
            float angle = (float)rng.uniform(-5.0, 5.0);

            resample(scaledImg, RotatedRect(center, size, angle), standardPatch);
            for( int y = 0; y < standardPatch.rows; y++ )
            {
                uchar* patchRow = standardPatch.ptr(y);
                for( int x = 0; x < standardPatch.cols; x++ )
                {
                    int newValue = patchRow[x] + cvRound(rng.gaussian(5.0));
                    patchRow[x] = saturate_cast<uchar>(newValue);
                }
            }
            examplesForModel.push_back(standardPatch);

#if defined BLUR_AS_VADIM
            GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0);
            resize(blurredPatch, blurredPatch, initSize_, 0, 0, INTER_LINEAR_EXACT);
#else
            resample(blurredImg, RotatedRect(center, size, angle), blurredPatch);
#endif
            examplesForEnsemble.push_back(blurredPatch);
        }
    }
    return 0;
}

bool TrackerTLDImpl::Nexpert::operator()(Rect2d box)
{
    if( overlap(resultBox_, box) < NEXPERT_THRESHOLD )
        return false;
    else
        return true;
}

Data::Data(Rect2d initBox)
{
    double minDim = std::min(initBox.width, initBox.height);
    scale = 20.0 / minDim;
    minSize.width = (int)(initBox.width * 20.0 / minDim);
    minSize.height = (int)(initBox.height * 20.0 / minDim);
    frameNum = 0;
}

void Data::printme(FILE*  port)
{
    dfprintf((port, "Data:\n"));
    dfprintf((port, "\tframeNum = %d\n", frameNum));
    dfprintf((port, "\tconfident = %s\n", confident?"true":"false"));
    dfprintf((port, "\tfailedLastTime = %s\n", failedLastTime?"true":"false"));
    dfprintf((port, "\tminSize = %dx%d\n", minSize.width, minSize.height));
}

}

}