Commit ced5aa76 authored by Andrej Muhič's avatar Andrej Muhič Committed by Alexander Alekhin

Merge pull request #1552 from amuhic:master

Implementation of CSR-DCF tracker (#1552)

* Initial commit for CSR-DCF tracker implementation

* Fixes for automatic build

* General code fixes

* Removed unused parameters. Added CSRT to automatic tests.

* Fixed VS build warnings. Fixed a bug with gray sequences.

* Fixed VS build errors for samples file.
parent 295dce52
......@@ -1462,6 +1462,71 @@ public:
//! @}
/*********************************** CSRT ************************************/
/** @brief Discriminative Correlation Filter Tracker with Channel and Spatial Reliability
*/
class CV_EXPORTS_W TrackerCSRT : public Tracker
{
public:
struct CV_EXPORTS Params
{
/**
* \brief Constructor
*/
Params();
/**
* \brief Read parameters from file
*/
void read(const FileNode& /*fn*/);
/**
* \brief Write parameters from file
*/
void write(cv::FileStorage& fs) const;
bool use_hog;
bool use_color_names;
bool use_gray;
bool use_rgb;
bool use_channel_weights;
bool use_segmentation;
std::string window_function; //!< Window function: "hann", "cheb", "kaiser"
float kaiser_alpha;
float cheb_attenuation;
float template_size;
float gsl_sigma;
float hog_orientations;
float hog_clip;
float padding;
float filter_lr;
float weights_lr;
int num_hog_channels_used;
int admm_iterations;
int histogram_bins;
float histogram_lr;
int background_ratio;
int number_of_scales;
float scale_sigma_factor;
float scale_model_max_area;
float scale_lr;
float scale_step;
};
/** @brief Constructor
@param parameters CSRT parameters TrackerCSRT::Params
*/
static Ptr<TrackerCSRT> create(const TrackerCSRT::Params &parameters);
CV_WRAP static Ptr<TrackerCSRT> create();
virtual void setInitialMask(const Mat mask) = 0;
virtual ~TrackerCSRT() {}
};
} /* namespace cv */
#endif
#include <opencv2/core/utility.hpp>
#include <opencv2/tracking.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
#include <cstring>
#include <fstream>
#include "samples_utility.hpp"
using namespace std;
using namespace cv;
int main(int argc, char** argv)
{
// show help
if (argc<2) {
cout <<
" Usage: example_tracking_csrt <video_name>\n"
" examples:\n"
" example_tracking_csrt Bolt/img/%04.jpg\n"
" example_tracking_csrt Bolt/img/%04.jpg Bolt/grouondtruth.txt\n"
" example_tracking_csrt faceocc2.webm\n"
<< endl;
return 0;
}
// create the tracker
Ptr<TrackerCSRT> tracker = TrackerCSRT::create();
// const char* param_file_path = "/home/amuhic/Workspace/3_dip/params.yml";
// FileStorage fs(params_file_path, FileStorage::WRITE);
// tracker->write(fs);
// FileStorage fs(param_file_path, FileStorage::READ);
// tracker->read( fs.root());
// set input video
std::string video = argv[1];
VideoCapture cap(video);
// and read first frame
Mat frame;
cap >> frame;
// target bounding box
Rect2d roi;
if (argc > 2) {
// read first line of ground-truth file
std::string groundtruthPath = argv[2];
std::ifstream gtIfstream(groundtruthPath.c_str());
std::string gtLine;
getline(gtIfstream, gtLine);
gtIfstream.close();
// parse the line by elements
std::stringstream gtStream(gtLine);
std::string element;
std::vector<int> elements;
while (std::getline(gtStream, element, ','))
{
elements.push_back(cvRound(std::atof(element.c_str())));
}
if (elements.size() == 4) {
// ground-truth is rectangle
roi = cv::Rect(elements[0], elements[1], elements[2], elements[3]);
}
else if (elements.size() == 8) {
// ground-truth is polygon
int xMin = cvRound(min(elements[0], min(elements[2], min(elements[4], elements[6]))));
int yMin = cvRound(min(elements[1], min(elements[3], min(elements[5], elements[7]))));
int xMax = cvRound(max(elements[0], max(elements[2], max(elements[4], elements[6]))));
int yMax = cvRound(max(elements[1], max(elements[3], max(elements[5], elements[7]))));
roi = cv::Rect(xMin, yMin, xMax - xMin, yMax - yMin);
// create mask from polygon and set it to the tracker
cv::Rect aaRect = cv::Rect(xMin, yMin, xMax - xMin, yMax - yMin);
cout << aaRect.size() << endl;
Mat mask = Mat::zeros(aaRect.size(), CV_8UC1);
const int n = 4;
std::vector<cv::Point> poly_points(n);
//Translate x and y to rects start position
int sx = aaRect.x;
int sy = aaRect.y;
for (int i = 0; i < n; ++i) {
poly_points[i] = Point(elements[2 * i] - sx, elements[2 * i + 1] - sy);
}
cv::fillConvexPoly(mask, poly_points, Scalar(1.0), 8);
mask.convertTo(mask, CV_32FC1);
tracker->setInitialMask(mask);
}
else {
std::cout << "Number of ground-truth elements is not 4 or 8." << std::endl;
}
}
else {
// second argument is not given - user selects target
roi = selectROI("tracker", frame, true, false);
}
//quit if ROI was not selected
if (roi.width == 0 || roi.height == 0)
return 0;
// initialize the tracker
int64 t1 = cv::getTickCount();
tracker->init(frame, roi);
int64 t2 = cv::getTickCount();
int64 tick_counter = t2 - t1;
// do the tracking
printf("Start the tracking process, press ESC to quit.\n");
int frame_idx = 1;
for (;;) {
// get frame from the video
cap >> frame;
// stop the program if no more images
if (frame.rows == 0 || frame.cols == 0)
break;
// update the tracking result
t1 = cv::getTickCount();
bool isfound = tracker->update(frame, roi);
t2 = cv::getTickCount();
tick_counter += t2 - t1;
frame_idx++;
if (!isfound) {
cout << "The target has been lost...\n";
waitKey(0);
return 0;
}
// draw the tracked object and show the image
rectangle(frame, roi, Scalar(255, 0, 0), 2, 1);
imshow("tracker", frame);
//quit on ESC button
if (waitKey(1) == 27)break;
}
cout << "Elapsed sec: " << static_cast<double>(tick_counter) / cv::getTickFrequency() << endl;
cout << "FPS: " << ((double)(frame_idx)) / (static_cast<double>(tick_counter) / cv::getTickFrequency()) << endl;
}
......@@ -21,6 +21,8 @@ inline cv::Ptr<cv::Tracker> createTrackerByName(cv::String name)
tracker = cv::TrackerGOTURN::create();
else if (name == "MOSSE")
tracker = cv::TrackerMOSSE::create();
else if (name == "CSRT")
tracker = cv::TrackerCSRT::create();
else
CV_Error(cv::Error::StsBadArg, "Invalid tracking algorithm name\n");
......
This diff is collapsed.
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "precomp.hpp"
#include "trackerCSRTScaleEstimation.hpp"
#include "trackerCSRTUtils.hpp"
//Discriminative Scale Space Tracking
namespace cv
{
class ParallelGetScaleFeatures : public ParallelLoopBody
{
public:
ParallelGetScaleFeatures(
Mat img,
Point2f pos,
Size2f base_target_sz,
float current_scale,
std::vector<float> &scale_factors,
Mat scale_window,
Size scale_model_sz,
int col_len,
Mat &result)
{
this->img = img;
this->pos = pos;
this->base_target_sz = base_target_sz;
this->current_scale = current_scale;
this->scale_factors = scale_factors;
this->scale_window = scale_window;
this->scale_model_sz = scale_model_sz;
this->col_len = col_len;
this->result = result;
}
virtual void operator ()(const Range& range) const
{
for (int s = range.start; s < range.end; s++) {
Size patch_sz = Size(static_cast<int>(current_scale * scale_factors[s] * base_target_sz.width),
static_cast<int>(current_scale * scale_factors[s] * base_target_sz.height));
Mat img_patch = get_subwindow(img, pos, patch_sz.width, patch_sz.height);
img_patch.convertTo(img_patch, CV_32FC3);
resize(img_patch, img_patch, Size(scale_model_sz.width, scale_model_sz.height),0,0,INTER_LINEAR);
std::vector<Mat> hog;
hog = get_features_hog(img_patch, 4);
for (int i = 0; i < static_cast<int>(hog.size()); ++i) {
hog[i] = hog[i].t();
hog[i] = scale_window.at<float>(0,s) * hog[i].reshape(0, col_len);
hog[i].copyTo(result(Rect(Point(s, i*col_len), hog[i].size())));
}
}
}
ParallelGetScaleFeatures& operator=(const ParallelGetScaleFeatures &) {
return *this;
}
private:
Mat img;
Point2f pos;
Size2f base_target_sz;
float current_scale;
std::vector<float> scale_factors;
Mat scale_window;
Size scale_model_sz;
int col_len;
Mat result;
};
DSST::DSST(const Mat &image,
Rect2f bounding_box,
Size2f template_size,
int numberOfScales,
float scaleStep,
float maxModelArea,
float sigmaFactor,
float scaleLearnRate):
scales_count(numberOfScales), scale_step(scaleStep), max_model_area(maxModelArea),
sigma_factor(sigmaFactor), learn_rate(scaleLearnRate)
{
original_targ_sz = bounding_box.size();
Point2f object_center = Point2f(bounding_box.x + original_targ_sz.width / 2,
bounding_box.y + original_targ_sz.height / 2);
current_scale_factor = 1.0;
if(scales_count % 2 == 0)
scales_count++;
scale_sigma = static_cast<float>(sqrt(scales_count) * sigma_factor);
min_scale_factor = pow(scale_step,
cvCeil(log(max(5.0 / template_size.width, 5.0 / template_size.height)) / log(scale_step)));
max_scale_factor = powf(scale_step,
static_cast<float>(cvFloor(log(min((float)image.rows / (float)bounding_box.width,
(float)image.cols / (float)bounding_box.height)) / log(scale_step))));
ys = Mat(1, scales_count, CV_32FC1);
float ss, sf;
for(int i = 0; i < ys.cols; ++i) {
ss = (float)(i+1) - cvCeil((float)scales_count / 2.0f);
ys.at<float>(0,i) = static_cast<float>(exp(-0.5 * pow(ss,2) / pow(scale_sigma,2)));
sf = static_cast<float>(i + 1);
scale_factors.push_back(pow(scale_step, cvCeil((float)scales_count / 2.0f) - sf));
}
scale_window = get_hann_win(Size(scales_count, 1));
float scale_model_factor = 1.0;
if(template_size.width * template_size.height * pow(scale_model_factor, 2) > max_model_area)
{
scale_model_factor = sqrt(max_model_area /
(template_size.width * template_size.height));
}
scale_model_sz = Size(cvFloor(template_size.width * scale_model_factor),
cvFloor(template_size.height * scale_model_factor));
Mat scale_resp = get_scale_features(image, object_center, original_targ_sz,
current_scale_factor, scale_factors, scale_window, scale_model_sz);
Mat ysf_row = Mat(ys.size(), CV_32FC2);
dft(ys, ysf_row, DFT_ROWS | DFT_COMPLEX_OUTPUT, 0);
ysf = repeat(ysf_row, scale_resp.rows, 1);
Mat Fscale_resp;
dft(scale_resp, Fscale_resp, DFT_ROWS | DFT_COMPLEX_OUTPUT);
mulSpectrums(ysf, Fscale_resp, sf_num, 0 , true);
Mat sf_den_all;
mulSpectrums(Fscale_resp, Fscale_resp, sf_den_all, 0, true);
reduce(sf_den_all, sf_den, 0, CV_REDUCE_SUM, -1);
}
DSST::~DSST()
{
}
Mat DSST::get_scale_features(
Mat img,
Point2f pos,
Size2f base_target_sz,
float current_scale,
std::vector<float> &scale_factors,
Mat scale_window,
Size scale_model_sz)
{
Mat result;
int col_len = 0;
Size patch_sz = Size(cvFloor(current_scale * scale_factors[0] * base_target_sz.width),
cvFloor(current_scale * scale_factors[0] * base_target_sz.height));
Mat img_patch = get_subwindow(img, pos, patch_sz.width, patch_sz.height);
img_patch.convertTo(img_patch, CV_32FC3);
resize(img_patch, img_patch, Size(scale_model_sz.width, scale_model_sz.height),0,0,INTER_LINEAR);
std::vector<Mat> hog;
hog = get_features_hog(img_patch, 4);
result = Mat(Size((int)scale_factors.size(), hog[0].cols * hog[0].rows * (int)hog.size()), CV_32F);
col_len = hog[0].cols * hog[0].rows;
for (int i = 0; i < static_cast<int>(hog.size()); ++i) {
hog[i] = hog[i].t();
hog[i] = scale_window.at<float>(0,0) * hog[i].reshape(0, col_len);
hog[i].copyTo(result(Rect(Point(0, i*col_len), hog[i].size())));
}
ParallelGetScaleFeatures parallelGetScaleFeatures(img, pos, base_target_sz,
current_scale, scale_factors, scale_window, scale_model_sz, col_len, result);
parallel_for_(Range(1, static_cast<int>(scale_factors.size())), parallelGetScaleFeatures);
return result;
}
void DSST::update(const Mat &image, const Point2f object_center)
{
Mat scale_features = get_scale_features(image, object_center, original_targ_sz,
current_scale_factor, scale_factors, scale_window, scale_model_sz);
Mat Fscale_features;
dft(scale_features, Fscale_features, DFT_ROWS | DFT_COMPLEX_OUTPUT);
Mat new_sf_num;
Mat new_sf_den;
Mat new_sf_den_all;
mulSpectrums(ysf, Fscale_features, new_sf_num, DFT_ROWS, true);
Mat sf_den_all;
mulSpectrums(Fscale_features, Fscale_features, new_sf_den_all, DFT_ROWS, true);
reduce(new_sf_den_all, new_sf_den, 0, CV_REDUCE_SUM, -1);
sf_num = (1 - learn_rate) * sf_num + learn_rate * new_sf_num;
sf_den = (1 - learn_rate) * sf_den + learn_rate * new_sf_den;
}
float DSST::getScale(const Mat &image, const Point2f object_center)
{
Mat scale_features = get_scale_features(image, object_center, original_targ_sz,
current_scale_factor, scale_factors, scale_window, scale_model_sz);
Mat Fscale_features;
dft(scale_features, Fscale_features, DFT_ROWS | DFT_COMPLEX_OUTPUT);
mulSpectrums(Fscale_features, sf_num, Fscale_features, 0, false);
Mat scale_resp;
reduce(Fscale_features, scale_resp, 0, CV_REDUCE_SUM, -1);
scale_resp = divide_complex_matrices(scale_resp, sf_den + 0.01f);
idft(scale_resp, scale_resp, DFT_REAL_OUTPUT|DFT_SCALE);
Point max_loc;
minMaxLoc(scale_resp, NULL, NULL, NULL, &max_loc);
current_scale_factor *= scale_factors[max_loc.x];
if(current_scale_factor < min_scale_factor)
current_scale_factor = min_scale_factor;
else if(current_scale_factor > max_scale_factor)
current_scale_factor = max_scale_factor;
return current_scale_factor;
}
} /* namespace cv */
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_TRACKER_CSRT_SCALE_ESTIMATION
#define OPENCV_TRACKER_CSRT_SCALE_ESTIMATION
namespace cv
{
class DSST {
public:
DSST() {};
DSST(const Mat &image, Rect2f bounding_box, Size2f template_size, int numberOfScales,
float scaleStep, float maxModelArea, float sigmaFactor, float scaleLearnRate);
~DSST();
void update(const Mat &image, const Point2f objectCenter);
float getScale(const Mat &image, const Point2f objecCenter);
private:
Mat get_scale_features(Mat img, Point2f pos, Size2f base_target_sz, float current_scale,
std::vector<float> &scale_factors, Mat scale_window, Size scale_model_sz);
Size scale_model_sz;
Mat ys;
Mat ysf;
Mat scale_window;
std::vector<float> scale_factors;
Mat sf_num;
Mat sf_den;
float scale_sigma;
float min_scale_factor;
float max_scale_factor;
float current_scale_factor;
int scales_count;
float scale_step;
float max_model_area;
float sigma_factor;
float learn_rate;
Size original_targ_sz;
};
} /* namespace cv */
#endif
This diff is collapsed.
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_TRACKER_CSRT_SEGMENTATION
#define OPENCV_TRACKER_CSRT_SEGMENTATION
namespace cv
{
class Histogram
{
public:
int m_numBinsPerDim;
int m_numDim;
Histogram() : m_numBinsPerDim(0), m_numDim(0) {}
Histogram(int numDimensions, int numBinsPerDimension = 8);
void extractForegroundHistogram(std::vector<cv::Mat> & imgChannels,
cv::Mat weights, bool useMatWeights, int x1, int y1, int x2, int y2);
void extractBackGroundHistogram(std::vector<cv::Mat> & imgChannels,
int x1, int y1, int x2, int y2, int outer_x1, int outer_y1,
int outer_x2, int outer_y2);
cv::Mat backProject(std::vector<cv::Mat> & imgChannels);
std::vector<double> getHistogramVector();
void setHistogramVector(double *vector);
private:
int p_size;
std::vector<double> p_bins;
std::vector<int> p_dimIdCoef;
inline double kernelProfile_Epanechnikov(double x)
{ return (x <= 1) ? (2.0/CV_PI)*(1-x) : 0; }
};
class Segment
{
public:
static std::pair<cv::Mat, cv::Mat> computePosteriors(std::vector<cv::Mat> & imgChannels,
int x1, int y1, int x2, int y2, cv::Mat weights, cv::Mat fgPrior,
cv::Mat bgPrior, const Histogram &fgHistPrior, int numBinsPerChannel = 16);
static std::pair<cv::Mat, cv::Mat> computePosteriors2(std::vector<cv::Mat> & imgChannels,
int x1, int y1, int x2, int y2, double p_b, cv::Mat fgPrior,
cv::Mat bgPrior, Histogram hist_target, Histogram hist_background);
static std::pair<cv::Mat, cv::Mat> computePosteriors2(std::vector<cv::Mat> &imgChannels,
cv::Mat fgPrior, cv::Mat bgPrior, Histogram hist_target, Histogram hist_background);
private:
static std::pair<cv::Mat, cv::Mat> getRegularizedSegmentation(cv::Mat & prob_o,
cv::Mat & prob_b, cv::Mat &prior_o, cv::Mat &prior_b);
inline static double gaussian(double x2, double y2, double std2){
return exp(-(x2 + y2)/(2*std2))/(2*CV_PI*std2);
}
};
}//cv namespace
#endif
This diff is collapsed.
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_TRACKER_CSRT_UTILS
#define OPENCV_TRACKER_CSRT_UTILS
#include <fstream>
#include <iostream>
#include <vector>
#include <iostream>
#include <algorithm>
#include <iterator>
namespace cv
{
inline int modul(int a, int b)
{
// function calculates the module of two numbers and it takes into account also negative numbers
return ((a % b) + b) % b;
}
inline double kernel_epan(double x)
{
return (x <= 1) ? (2.0/3.14)*(1-x) : 0;
}
Mat circshift(Mat matrix, int dx, int dy);
Mat gaussian_shaped_labels(const float sigma, const int w, const int h);
std::vector<Mat> fourier_transform_features(const std::vector<Mat> &M);
Mat divide_complex_matrices(const Mat &A, const Mat &B);
Mat get_subwindow(const Mat &image, const Point2f center,
const int w, const int h,Rect *valid_pixels = NULL);
float subpixel_peak(const Mat &response, const std::string &s, const Point2f &p);
double get_max(const Mat &m);
double get_min(const Mat &m);
Mat get_hann_win(Size sz);
Mat get_kaiser_win(Size sz, float alpha);
Mat get_chebyshev_win(Size sz, float attenuation);
std::vector<Mat> get_features_rgb(const Mat &patch, const Size &output_size);
std::vector<Mat> get_features_hog(const Mat &im, const int bin_size);
std::vector<Mat> get_features_cn(const Mat &im, const Size &output_size);
Mat bgr2hsv(const Mat &img);
} //cv namespace
#endif
......@@ -467,6 +467,12 @@ TEST_P(DistanceAndOverlap, MOSSE)
test.run();
}
TEST_P(DistanceAndOverlap, CSRT)
{
TrackerTest test( TrackerCSRT::create(), dataset, 22, .7f, NoTransform);
test.run();
}
/***************************************************************************************/
//Tests with shifted initial window
TEST_P(DistanceAndOverlap, Shifted_Data_MedianFlow)
......@@ -504,6 +510,12 @@ TEST_P(DistanceAndOverlap, Shifted_Data_MOSSE)
TrackerTest test( TrackerMOSSE::create(), dataset, 13, .69f, CenterShiftLeft);
test.run();
}
TEST_P(DistanceAndOverlap, Shifted_Data_CSRT)
{
TrackerTest test( TrackerCSRT::create(), dataset, 13, .69f, CenterShiftLeft);
test.run();
}
/***************************************************************************************/
//Tests with scaled initial window
TEST_P(DistanceAndOverlap, Scaled_Data_MedianFlow)
......@@ -549,6 +561,12 @@ TEST_P(DistanceAndOverlap, Scaled_Data_MOSSE)
test.run();
}
TEST_P(DistanceAndOverlap, Scaled_Data_CSRT)
{
TrackerTest test( TrackerCSRT::create(), dataset, 22, 0.69f, Scale_1_1, 1);
test.run();
}
INSTANTIATE_TEST_CASE_P( Tracking, DistanceAndOverlap, TESTSET_NAMES);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment