/*M/////////////////////////////////////////////////////////////////////////////////////// // // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. // // By downloading, copying, installing or using the software you agree to this license. // If you do not agree to this license, do not download, install, // copy or use the software. // // // License Agreement // For Open Source Computer Vision Library // // Copyright (C) 2013, OpenCV Foundation, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, // are permitted provided that the following conditions are met: // // * Redistribution's of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // * Redistribution's in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // // * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and // any express or implied warranties, including, but not limited to, the implied // warranties of merchantability and fitness for a particular purpose are disclaimed. // In no event shall the Intel Corporation or contributors be liable for any direct, // indirect, incidental, special, exemplary, or consequential damages // (including, but not limited to, procurement of substitute goods or services; // loss of use, data, or profits; or business interruption) however caused // and on any theory of liability, whether in contract, strict liability, // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // //M*/ #include "tldModel.hpp" namespace cv { namespace tld { //Constructor TrackerTLDModel::TrackerTLDModel(TrackerTLD::Params params, const Mat& image, const Rect2d& boundingBox, Size minSize): timeStampPositiveNext(0), timeStampNegativeNext(0), minSize_(minSize), params_(params), boundingBox_(boundingBox) { std::vector<Rect2d> closest, scanGrid; Mat scaledImg, blurredImg, image_blurred; //Create Detector detector = Ptr<TLDDetector>(new TLDDetector()); //Propagate data to Detector posNum = 0; negNum = 0; posExp = Mat(Size(225, 500), CV_8UC1); negExp = Mat(Size(225, 500), CV_8UC1); detector->posNum = &posNum; detector->negNum = &negNum; detector->posExp = &posExp; detector->negExp = &negExp; detector->positiveExamples = &positiveExamples; detector->negativeExamples = &negativeExamples; detector->timeStampsPositive = &timeStampsPositive; detector->timeStampsNegative = &timeStampsNegative; detector->originalVariancePtr = &originalVariance_; //Calculate the variance in initial BB originalVariance_ = variance(image(boundingBox)); //Find the scale double scale = scaleAndBlur(image, cvRound(log(1.0 * boundingBox.width / (minSize.width)) / log(SCALE_STEP)), scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP); GaussianBlur(image, image_blurred, GaussBlurKernelSize, 0.0); TLDDetector::generateScanGrid(image.rows, image.cols, minSize_, scanGrid); getClosestN(scanGrid, Rect2d(boundingBox.x / scale, boundingBox.y / scale, boundingBox.width / scale, boundingBox.height / scale), 10, closest); Mat_<uchar> blurredPatch(minSize); TLDEnsembleClassifier::makeClassifiers(minSize, MEASURES_PER_CLASSIFIER, GRIDSIZE, detector->classifiers); //Generate initial positive samples and put them to the model positiveExamples.reserve(200); for (int i = 0; i < (int)closest.size(); i++) { for (int j = 0; j < 20; j++) { Point2f center; Size2f size; Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); center.x = (float)(closest[i].x + closest[i].width * (0.5 + rng.uniform(-0.01, 0.01))); center.y = (float)(closest[i].y + closest[i].height * (0.5 + rng.uniform(-0.01, 0.01))); size.width = (float)(closest[i].width * rng.uniform((double)0.99, (double)1.01)); size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01)); float angle = (float)rng.uniform(-10.0, 10.0); resample(scaledImg, RotatedRect(center, size, angle), standardPatch); for (int y = 0; y < standardPatch.rows; y++) { for (int x = 0; x < standardPatch.cols; x++) { standardPatch(x, y) += (uchar)rng.gaussian(5.0); } } #ifdef BLUR_AS_VADIM GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0); resize(blurredPatch, blurredPatch, minSize); #else resample(blurredImg, RotatedRect(center, size, angle), blurredPatch); #endif pushIntoModel(standardPatch, true); for (int k = 0; k < (int)detector->classifiers.size(); k++) detector->classifiers[k].integrate(blurredPatch, true); } } //Generate initial negative samples and put them to the model TLDDetector::generateScanGrid(image.rows, image.cols, minSize, scanGrid, true); negativeExamples.clear(); negativeExamples.reserve(NEG_EXAMPLES_IN_INIT_MODEL); std::vector<int> indices; indices.reserve(NEG_EXAMPLES_IN_INIT_MODEL); while ((int)negativeExamples.size() < NEG_EXAMPLES_IN_INIT_MODEL) { int i = rng.uniform((int)0, (int)scanGrid.size()); if (std::find(indices.begin(), indices.end(), i) == indices.end() && overlap(boundingBox, scanGrid[i]) < NEXPERT_THRESHOLD) { Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE); resample(image, scanGrid[i], standardPatch); pushIntoModel(standardPatch, false); resample(image_blurred, scanGrid[i], blurredPatch); for (int k = 0; k < (int)detector->classifiers.size(); k++) detector->classifiers[k].integrate(blurredPatch, false); } } } void TrackerTLDModel::integrateRelabeled(Mat& img, Mat& imgBlurred, const std::vector<TLDDetector::LabeledPatch>& patches) { Mat_<uchar> standardPatch(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE), blurredPatch(minSize_); int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; for (int k = 0; k < (int)patches.size(); k++) { if (patches[k].shouldBeIntegrated) { resample(img, patches[k].rect, standardPatch); if (patches[k].isObject) { positiveIntoModel++; pushIntoModel(standardPatch, true); } else { negativeIntoModel++; pushIntoModel(standardPatch, false); } } #ifdef CLOSED_LOOP if (patches[k].shouldBeIntegrated || !patches[k].isPositive) #else if (patches[k].shouldBeIntegrated) #endif { resample(imgBlurred, patches[k].rect, blurredPatch); if (patches[k].isObject) positiveIntoEnsemble++; else negativeIntoEnsemble++; for (int i = 0; i < (int)detector->classifiers.size(); i++) detector->classifiers[i].integrate(blurredPatch, patches[k].isObject); } } } void TrackerTLDModel::integrateAdditional(const std::vector<Mat_<uchar> >& eForModel, const std::vector<Mat_<uchar> >& eForEnsemble, bool isPositive) { int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; if ((int)eForModel.size() == 0) return; for (int k = 0; k < (int)eForModel.size(); k++) { double sr = detector->Sr(eForModel[k]); if ((sr > THETA_NN) != isPositive) { if (isPositive) { positiveIntoModel++; pushIntoModel(eForModel[k], true); } else { negativeIntoModel++; pushIntoModel(eForModel[k], false); } } double p = 0; for (int i = 0; i < (int)detector->classifiers.size(); i++) p += detector->classifiers[i].posteriorProbability(eForEnsemble[k].data, (int)eForEnsemble[k].step[0]); p /= detector->classifiers.size(); if ((p > ENSEMBLE_THRESHOLD) != isPositive) { if (isPositive) positiveIntoEnsemble++; else negativeIntoEnsemble++; for (int i = 0; i < (int)detector->classifiers.size(); i++) detector->classifiers[i].integrate(eForEnsemble[k], isPositive); } } } void TrackerTLDModel::ocl_integrateAdditional(const std::vector<Mat_<uchar> >& eForModel, const std::vector<Mat_<uchar> >& eForEnsemble, bool isPositive) { int positiveIntoModel = 0, negativeIntoModel = 0, positiveIntoEnsemble = 0, negativeIntoEnsemble = 0; if ((int)eForModel.size() == 0) return; //Prepare batch of patches int numOfPatches = (int)eForModel.size(); Mat_<uchar> stdPatches(numOfPatches, 225); double *resultSr = new double[numOfPatches]; double *resultSc = new double[numOfPatches]; uchar *patchesData = stdPatches.data; for (int i = 0; i < numOfPatches; i++) { uchar *stdPatchData = eForModel[i].data; for (int j = 0; j < 225; j++) patchesData[225 * i + j] = stdPatchData[j]; } //Calculate Sr and Sc batches detector->ocl_batchSrSc(stdPatches, resultSr, resultSc, numOfPatches); for (int k = 0; k < (int)eForModel.size(); k++) { double sr = resultSr[k]; if ((sr > THETA_NN) != isPositive) { if (isPositive) { positiveIntoModel++; pushIntoModel(eForModel[k], true); } else { negativeIntoModel++; pushIntoModel(eForModel[k], false); } } double p = 0; for (int i = 0; i < (int)detector->classifiers.size(); i++) p += detector->classifiers[i].posteriorProbability(eForEnsemble[k].data, (int)eForEnsemble[k].step[0]); p /= detector->classifiers.size(); if ((p > ENSEMBLE_THRESHOLD) != isPositive) { if (isPositive) positiveIntoEnsemble++; else negativeIntoEnsemble++; for (int i = 0; i < (int)detector->classifiers.size(); i++) detector->classifiers[i].integrate(eForEnsemble[k], isPositive); } } } //Push the patch to the model void TrackerTLDModel::pushIntoModel(const Mat_<uchar>& example, bool positive) { std::vector<Mat_<uchar> >* proxyV; int* proxyN; std::vector<int>* proxyT; if (positive) { if (posNum < 500) { uchar *patchPtr = example.data; uchar *modelPtr = posExp.data; for (int i = 0; i < STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE; i++) modelPtr[posNum*STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE + i] = patchPtr[i]; posNum++; } proxyV = &positiveExamples; proxyN = &timeStampPositiveNext; proxyT = &timeStampsPositive; } else { if (negNum < 500) { uchar *patchPtr = example.data; uchar *modelPtr = negExp.data; for (int i = 0; i < STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE; i++) modelPtr[negNum*STANDARD_PATCH_SIZE*STANDARD_PATCH_SIZE + i] = patchPtr[i]; negNum++; } proxyV = &negativeExamples; proxyN = &timeStampNegativeNext; proxyT = &timeStampsNegative; } if ((int)proxyV->size() < MAX_EXAMPLES_IN_MODEL) { proxyV->push_back(example); proxyT->push_back(*proxyN); } else { int index = rng.uniform((int)0, (int)proxyV->size()); (*proxyV)[index] = example; (*proxyT)[index] = (*proxyN); } (*proxyN)++; } void TrackerTLDModel::printme(FILE* port) { dfprintf((port, "TrackerTLDModel:\n")); dfprintf((port, "\tpositiveExamples.size() = %d\n", (int)positiveExamples.size())); dfprintf((port, "\tnegativeExamples.size() = %d\n", (int)negativeExamples.size())); } } }