Commit 40f1ac51 authored by szk1509's avatar szk1509 Committed by Alexander Alekhin

Merge pull request #973 from szk1509:master

new corner refinement method :: using the contour-lines (#973)

* doCornerRefinement to CornerRefinementMethod :: detected contours points are used to detect the corners

* some little corrections

* samples edited

* documented :)

* tabs corrected

* Docu corrections

* refinement for all candidates

* refinement for all candidates :: copy paste error corrected

* comment
parent 4ed8c5eb
......@@ -76,7 +76,11 @@ namespace aruco {
//! @addtogroup aruco
//! @{
enum CornerRefineMethod{
CORNER_REFINE_NONE, // default corners
CORNER_REFINE_SUBPIX, // refine the corners using subpix
CORNER_REFINE_CONTOUR // refine the corners using the contour-points
};
/**
* @brief Parameters for the detectMarker process:
......@@ -100,7 +104,8 @@ namespace aruco {
* - minMarkerDistanceRate: minimum mean distance beetween two marker corners to be considered
* similar, so that the smaller one is removed. The rate is relative to the smaller perimeter
* of the two markers (default 0.05).
* - doCornerRefinement: do subpixel refinement or not
* - cornerRefinementMethod: corner refinement method. (CORNER_REFINE_NONE, no refinement.
* CORNER_REFINE_SUBPIX, do subpixel refinement. CORNER_REFINE_CONTOUR use contour-Points)
* - cornerRefinementWinSize: window size for the corner refinement process (in pixels) (default 5).
* - cornerRefinementMaxIterations: maximum number of iterations for stop criteria of the corner
* refinement process (default 30).
......@@ -137,7 +142,7 @@ struct CV_EXPORTS_W DetectorParameters {
CV_PROP_RW double minCornerDistanceRate;
CV_PROP_RW int minDistanceToBorder;
CV_PROP_RW double minMarkerDistanceRate;
CV_PROP_RW bool doCornerRefinement;
CV_PROP_RW int cornerRefinementMethod;
CV_PROP_RW int cornerRefinementWinSize;
CV_PROP_RW int cornerRefinementMaxIterations;
CV_PROP_RW double cornerRefinementMinAccuracy;
......@@ -165,6 +170,10 @@ struct CV_EXPORTS_W DetectorParameters {
* @param parameters marker detection parameters
* @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a
* correct codification. Useful for debugging purposes.
* @param cameraMatrix optional input 3x3 floating-point camera matrix
* \f$A = \vecthreethree{f_x}{0}{c_x}{0}{f_y}{c_y}{0}{0}{1}\f$
* @param distCoeff optional vector of distortion coefficients
* \f$(k_1, k_2, p_1, p_2[, k_3[, k_4, k_5, k_6],[s_1, s_2, s_3, s_4]])\f$ of 4, 5, 8 or 12 elements
*
* Performs marker detection in the input image. Only markers included in the specific dictionary
* are searched. For each detected marker, it returns the 2D position of its corner in the image
......@@ -175,7 +184,7 @@ struct CV_EXPORTS_W DetectorParameters {
*/
CV_EXPORTS_W void detectMarkers(InputArray image, const Ptr<Dictionary> &dictionary, OutputArrayOfArrays corners,
OutputArray ids, const Ptr<DetectorParameters> &parameters = DetectorParameters::create(),
OutputArrayOfArrays rejectedImgPoints = noArray());
OutputArrayOfArrays rejectedImgPoints = noArray(), InputArray cameraMatrix= noArray(), InputArray distCoeff= noArray());
......@@ -196,6 +205,7 @@ CV_EXPORTS_W void detectMarkers(InputArray image, const Ptr<Dictionary> &diction
* Each element in rvecs corresponds to the specific marker in imgPoints.
* @param tvecs array of output translation vectors (e.g. std::vector<cv::Vec3d>).
* Each element in tvecs corresponds to the specific marker in imgPoints.
* @param _objPoints array of object points of all the marker corners
*
* This function receives the detected markers and returns their pose estimation respect to
* the camera individually. So for each marker, one rotation and translation vector is returned.
......@@ -209,7 +219,7 @@ CV_EXPORTS_W void detectMarkers(InputArray image, const Ptr<Dictionary> &diction
*/
CV_EXPORTS_W void estimatePoseSingleMarkers(InputArrayOfArrays corners, float markerLength,
InputArray cameraMatrix, InputArray distCoeffs,
OutputArray rvecs, OutputArray tvecs);
OutputArray rvecs, OutputArray tvecs, OutputArray _objPoints = noArray());
......
......@@ -90,7 +90,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......
......@@ -90,7 +90,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......
......@@ -93,7 +93,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......@@ -145,7 +145,7 @@ int main(int argc, char *argv[]) {
return 0;
}
}
detectorParams->doCornerRefinement = true; // do corner refinement in markers
detectorParams->cornerRefinementMethod = aruco::CORNER_REFINE_SUBPIX; // do corner refinement in markers
String video;
if(parser.has("v")) {
......
......@@ -93,7 +93,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......
......@@ -94,7 +94,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......
......@@ -88,7 +88,7 @@ static bool readDetectorParameters(string filename, Ptr<aruco::DetectorParameter
fs["minCornerDistanceRate"] >> params->minCornerDistanceRate;
fs["minDistanceToBorder"] >> params->minDistanceToBorder;
fs["minMarkerDistanceRate"] >> params->minMarkerDistanceRate;
fs["doCornerRefinement"] >> params->doCornerRefinement;
fs["cornerRefinementMethod"] >> params->cornerRefinementMethod;
fs["cornerRefinementWinSize"] >> params->cornerRefinementWinSize;
fs["cornerRefinementMaxIterations"] >> params->cornerRefinementMaxIterations;
fs["cornerRefinementMinAccuracy"] >> params->cornerRefinementMinAccuracy;
......@@ -127,7 +127,7 @@ int main(int argc, char *argv[]) {
return 0;
}
}
detectorParams->doCornerRefinement = true; // do corner refinement in markers
detectorParams->cornerRefinementMethod = aruco::CORNER_REFINE_SUBPIX; // do corner refinement in markers
int camId = parser.get<int>("ci");
......
......@@ -63,7 +63,7 @@ DetectorParameters::DetectorParameters()
minCornerDistanceRate(0.05),
minDistanceToBorder(3),
minMarkerDistanceRate(0.05),
doCornerRefinement(false),
cornerRefinementMethod(CORNER_REFINE_NONE),
cornerRefinementWinSize(5),
cornerRefinementMaxIterations(30),
cornerRefinementMinAccuracy(0.1),
......@@ -602,7 +602,7 @@ static void _copyVector2Output(vector< vector< Point2f > > &vec, OutputArrayOfAr
* @brief Identify square candidates according to a marker dictionary
*/
static void _identifyCandidates(InputArray _image, vector< vector< Point2f > >& _candidates,
InputArrayOfArrays _contours, const Ptr<Dictionary> &_dictionary,
vector< vector<Point> >& _contours, const Ptr<Dictionary> &_dictionary,
vector< vector< Point2f > >& _accepted, vector< int >& ids,
const Ptr<DetectorParameters> &params,
OutputArrayOfArrays _rejected = noArray()) {
......@@ -612,6 +612,8 @@ static void _identifyCandidates(InputArray _image, vector< vector< Point2f > >&
vector< vector< Point2f > > accepted;
vector< vector< Point2f > > rejected;
vector< vector< Point > > contours;
CV_Assert(_image.getMat().total() != 0);
Mat grey;
......@@ -639,6 +641,9 @@ static void _identifyCandidates(InputArray _image, vector< vector< Point2f > >&
if(validCandidates[i] == 1) {
accepted.push_back(_candidates[i]);
ids.push_back(idsTmp[i]);
contours.push_back(_contours[i]);
} else {
rejected.push_back(_candidates[i]);
}
......@@ -647,6 +652,8 @@ static void _identifyCandidates(InputArray _image, vector< vector< Point2f > >&
// parse output
_accepted = accepted;
_contours= contours;
if(_rejected.needed()) {
_copyVector2Output(rejected, _rejected);
}
......@@ -656,7 +663,7 @@ static void _identifyCandidates(InputArray _image, vector< vector< Point2f > >&
/**
* @brief Final filter of markers after its identification
*/
static void _filterDetectedMarkers(vector< vector< Point2f > >& _corners, vector< int >& _ids) {
static void _filterDetectedMarkers(vector< vector< Point2f > >& _corners, vector< int >& _ids, vector< vector< Point> >& _contours) {
CV_Assert(_corners.size() == _ids.size());
if(_corners.empty()) return;
......@@ -707,15 +714,21 @@ static void _filterDetectedMarkers(vector< vector< Point2f > >& _corners, vector
vector< vector< Point2f > >::iterator filteredCorners = _corners.begin();
vector< int >::iterator filteredIds = _ids.begin();
vector< vector< Point > >::iterator filteredContours = _contours.begin();
for(unsigned int i = 0; i < toRemove.size(); i++) {
if(!toRemove[i]) {
*filteredCorners++ = _corners[i];
*filteredIds++ = _ids[i];
*filteredContours++ = _contours[i];
}
}
_ids.erase(filteredIds, _ids.end());
_corners.erase(filteredCorners, _corners.end());
_contours.erase(filteredContours, _contours.end());
}
}
......@@ -771,13 +784,183 @@ class MarkerSubpixelParallel : public ParallelLoopBody {
const Ptr<DetectorParameters> &params;
};
/**
* Line fitting A * B = C :: Called from function refineCandidateLines
* @param nContours, contour-container
*/
static Point3f _interpolate2Dline(const std::vector<cv::Point2f>& nContours){
float minX, minY, maxX, maxY;
minX = maxX = nContours[0].x;
minY = maxY = nContours[0].y;
for(unsigned int i = 0; i< nContours.size(); i++){
minX = nContours[i].x < minX ? nContours[i].x : minX;
minY = nContours[i].y < minY ? nContours[i].y : minY;
maxX = nContours[i].x > maxX ? nContours[i].x : maxX;
maxY = nContours[i].y > maxY ? nContours[i].y : maxY;
}
Mat A = Mat::ones((int)nContours.size(), 2, CV_32F); // Coefficient Matrix (N x 2)
Mat B((int)nContours.size(), 1, CV_32F); // Variables Matrix (N x 1)
Mat C; // Constant
if(maxX - minX > maxY - minY){
for(unsigned int i =0; i < nContours.size(); i++){
A.at<float>(i,0)= nContours[i].x;
B.at<float>(i,0)= nContours[i].y;
}
solve(A, B, C, DECOMP_NORMAL);
return Point3f(C.at<float>(0, 0), -1., C.at<float>(1, 0));
}
else{
for(unsigned int i =0; i < nContours.size(); i++){
A.at<float>(i,0)= nContours[i].y;
B.at<float>(i,0)= nContours[i].x;
}
solve(A, B, C, DECOMP_NORMAL);
return Point3f(-1., C.at<float>(0, 0), C.at<float>(1, 0));
}
}
/**
* Find the Point where the lines crosses :: Called from function refineCandidateLines
* @param nLine1
* @param nLine2
* @return Crossed Point
*/
static Point2f _getCrossPoint(Point3f nLine1, Point3f nLine2){
Matx22f A(nLine1.x, nLine1.y, nLine2.x, nLine2.y);
Vec2f B(-nLine1.z, -nLine2.z);
return Vec2f(A.solve(B).val);
}
static void _distortPoints(vector<cv::Point2f>& in, const Mat& camMatrix, const Mat& distCoeff) {
// trivial extrinsics
Matx31f Rvec(0,0,0);
Matx31f Tvec(0,0,0);
// calculate 3d points and then reproject, so opencv makes the distortion internally
vector<cv::Point3f> cornersPoints3d;
for (unsigned int i = 0; i < in.size(); i++){
float x= (in[i].x - float(camMatrix.at<double>(0, 2))) / float(camMatrix.at<double>(0, 0));
float y= (in[i].y - float(camMatrix.at<double>(1, 2))) / float(camMatrix.at<double>(1, 1));
cornersPoints3d.push_back(Point3f(x,y,1));
}
cv::projectPoints(cornersPoints3d, Rvec, Tvec, camMatrix, distCoeff, in);
}
/**
* Refine Corners using the contour vector :: Called from function detectMarkers
* @param nContours, contour-container
* @param nCorners, candidate Corners
* @param camMatrix, cameraMatrix input 3x3 floating-point camera matrix
* @param distCoeff, distCoeffs vector of distortion coefficient
*/
static void _refineCandidateLines(std::vector<Point>& nContours, std::vector<Point2f>& nCorners, const Mat& camMatrix, const Mat& distCoeff){
vector<Point2f> contour2f(nContours.begin(), nContours.end());
if(!camMatrix.empty() && !distCoeff.empty()){
undistortPoints(contour2f, contour2f, camMatrix, distCoeff);
}
/* 5 groups :: to group the edges
* 4 - classified by its corner
* extra group - (temporary) if contours do not begin with a corner
*/
vector<Point2f> cntPts[5];
int cornerIndex[4]={-1};
int group=4;
for ( unsigned int i =0; i < nContours.size(); i++ ) {
for(unsigned int j=0; j<4; j++){
if ( nCorners[j] == contour2f[i] ){
cornerIndex[j] = i;
group=j;
}
}
cntPts[group].push_back(contour2f[i]);
}
// saves extra group into corresponding
if( !cntPts[4].empty() ){
for( unsigned int i=0; i < cntPts[4].size() ; i++ )
cntPts[group].push_back(cntPts[4].at(i));
cntPts[4].clear();
}
//Evaluate contour direction :: using the position of the detected corners
int inc=1;
inc = ( (cornerIndex[0] > cornerIndex[1]) && (cornerIndex[3] > cornerIndex[0]) ) ? -1:inc;
inc = ( (cornerIndex[2] > cornerIndex[3]) && (cornerIndex[1] > cornerIndex[2]) ) ? -1:inc;
// calculate the line :: who passes through the grouped points
Point3f lines[4];
for(int i=0; i<4; i++){
lines[i]=_interpolate2Dline(cntPts[i]);
}
/*
* calculate the corner :: where the lines crosses to each other
* clockwise direction no clockwise direction
* 0 1
* .---. 1 .---. 2
* | | | |
* 3 .___. 0 .___.
* 2 3
*/
for(int i=0; i < 4; i++){
if(inc<0)
nCorners[i] = _getCrossPoint(lines[ i ], lines[ (i+1)%4 ]); // 01 12 23 30
else
nCorners[i] = _getCrossPoint(lines[ i ], lines[ (i+3)%4 ]); // 30 01 12 23
}
if(!camMatrix.empty() && !distCoeff.empty()){
_distortPoints(nCorners, camMatrix, distCoeff);
}
}
/**
* ParallelLoopBody class for the parallelization of the marker corner contour refinement
* Called from function detectMarkers()
*/
class MarkerContourParallel : public ParallelLoopBody {
public:
MarkerContourParallel( vector< vector< Point > >& _contours, vector< vector< Point2f > >& _candidates, const Mat& _camMatrix, const Mat& _distCoeff)
: contours(_contours), candidates(_candidates), camMatrix(_camMatrix), distCoeff(_distCoeff){}
void operator()(const Range &range) const {
for(int i = range.start; i < range.end; i++) {
_refineCandidateLines(contours[i], candidates[i], camMatrix, distCoeff);
}
}
private:
MarkerContourParallel &operator=(const MarkerContourParallel &){
return *this;
}
vector< vector< Point > >& contours;
vector< vector< Point2f > >& candidates;
const Mat& camMatrix;
const Mat& distCoeff;
};
/**
*/
void detectMarkers(InputArray _image, const Ptr<Dictionary> &_dictionary, OutputArrayOfArrays _corners,
OutputArray _ids, const Ptr<DetectorParameters> &_params,
OutputArrayOfArrays _rejectedImgPoints) {
OutputArrayOfArrays _rejectedImgPoints, InputArrayOfArrays camMatrix, InputArrayOfArrays distCoeff) {
CV_Assert(!_image.empty());
......@@ -795,14 +978,14 @@ void detectMarkers(InputArray _image, const Ptr<Dictionary> &_dictionary, Output
_rejectedImgPoints);
/// STEP 3: Filter detected markers;
_filterDetectedMarkers(candidates, ids);
_filterDetectedMarkers(candidates, ids, contours);
// copy to output arrays
_copyVector2Output(candidates, _corners);
Mat(ids).copyTo(_ids);
/// STEP 4: Corner refinement
if(_params->doCornerRefinement) {
/// STEP 4: Corner refinement :: use corner subpix
if( _params->cornerRefinementMethod == CORNER_REFINE_SUBPIX ) {
CV_Assert(_params->cornerRefinementWinSize > 0 && _params->cornerRefinementMaxIterations > 0 &&
_params->cornerRefinementMinAccuracy > 0);
......@@ -819,6 +1002,19 @@ void detectMarkers(InputArray _image, const Ptr<Dictionary> &_dictionary, Output
parallel_for_(Range(0, _corners.cols()),
MarkerSubpixelParallel(&grey, _corners, _params));
}
/// STEP 4, Optional : Corner refinement :: use contour container
if( _params->cornerRefinementMethod == CORNER_REFINE_CONTOUR){
if(! _ids.empty()){
// do corner refinement using the contours for each detected markers
parallel_for_(Range(0, _corners.cols()), MarkerContourParallel(contours, candidates, camMatrix.getMat(), distCoeff.getMat()));
// copy the corners to the output array
_copyVector2Output(candidates, _corners);
}
}
}
......@@ -861,7 +1057,7 @@ class SinglePoseEstimationParallel : public ParallelLoopBody {
*/
void estimatePoseSingleMarkers(InputArrayOfArrays _corners, float markerLength,
InputArray _cameraMatrix, InputArray _distCoeffs,
OutputArray _rvecs, OutputArray _tvecs) {
OutputArray _rvecs, OutputArray _tvecs, OutputArray _objPoints) {
CV_Assert(markerLength > 0);
......@@ -883,6 +1079,9 @@ void estimatePoseSingleMarkers(InputArrayOfArrays _corners, float markerLength,
parallel_for_(Range(0, nMarkers),
SinglePoseEstimationParallel(markerObjPoints, _corners, _cameraMatrix,
_distCoeffs, rvecs, tvecs));
if(_objPoints.needed()){
markerObjPoints.convertTo(_objPoints, -1);
}
}
......@@ -1161,7 +1360,7 @@ void refineDetectedMarkers(InputArray _image, const Ptr<Board> &_board,
if(closestCandidateIdx >= 0) {
// subpixel refinement
if(params.doCornerRefinement) {
if(_params->cornerRefinementMethod == CORNER_REFINE_SUBPIX) {
CV_Assert(params.cornerRefinementWinSize > 0 &&
params.cornerRefinementMaxIterations > 0 &&
params.cornerRefinementMinAccuracy > 0);
......
......@@ -299,7 +299,7 @@ void CV_ArucoRefine::run(int) {
vector< int > ids;
Ptr<aruco::DetectorParameters> params = aruco::DetectorParameters::create();
params->minDistanceToBorder = 3;
params->doCornerRefinement = true;
params->cornerRefinementMethod = aruco::CORNER_REFINE_SUBPIX;
params->markerBorderBits = markerBorder;
aruco::detectMarkers(img, dictionary, corners, ids, params, rejected);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment