Commit bc5d9c51 authored by Vadim Pisarevsky's avatar Vadim Pisarevsky

Merge pull request #92 from Bellaktris/inpainting

Inpainting
parents f07e304d 63c308a9
...@@ -6,6 +6,8 @@ ...@@ -6,6 +6,8 @@
#include "opencv2/core/utility.hpp" #include "opencv2/core/utility.hpp"
#include "opencv2/imgproc/types_c.h" #include "opencv2/imgproc/types_c.h"
#include <ctime>
#include <iostream>
const char* keys = const char* keys =
{ {
...@@ -54,9 +56,15 @@ int main( int argc, const char** argv ) ...@@ -54,9 +56,15 @@ int main( int argc, const char** argv )
printf( "Cannot read image file: %s\n", maskFilename.c_str() ); printf( "Cannot read image file: %s\n", maskFilename.c_str() );
return -1; return -1;
} }
cv::threshold(mask, mask, 128, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
cv::Mat res(src.size(), src.type()); cv::Mat res(src.size(), src.type());
int time = clock();
cv::xphoto::inpaint( src, mask, res, cv::xphoto::INPAINT_SHIFTMAP ); cv::xphoto::inpaint( src, mask, res, cv::xphoto::INPAINT_SHIFTMAP );
std::cout << "time = " << (clock() - time)
/ double(CLOCKS_PER_SEC) << std::endl;
cv::cvtColor(res, res, CV_Lab2RGB); cv::cvtColor(res, res, CV_Lab2RGB);
if ( outFilename == "" ) if ( outFilename == "" )
......
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __ADVANCED_TYPES_HPP__
#define __ADVANCED_TYPES_HPP__
#ifdef __cplusplus
#include <opencv2/core.hpp>
/********************* Functions *********************/
namespace cv
{
template <typename _Tp, typename _Tp2> static inline
cv::Size_<_Tp> operator * (const _Tp2 x, const cv::Size_<_Tp> &sz)
{
return cv::Size_<_Tp>(cv::saturate_cast<_Tp>(x*sz.width), cv::saturate_cast<_Tp>(x*sz.height));
}
template <typename _Tp, typename _Tp2> static inline
cv::Size_<_Tp> operator / (const cv::Size_<_Tp> &sz, const _Tp2 x)
{
return cv::Size_<_Tp>(cv::saturate_cast<_Tp>(sz.width/x), cv::saturate_cast<_Tp>(sz.height/x));
}
} // cv
#endif
#endif /* __ADVANCED_TYPES_HPP__ */
\ No newline at end of file
...@@ -40,6 +40,16 @@ ...@@ -40,6 +40,16 @@
#ifndef __ANNF_HPP__ #ifndef __ANNF_HPP__
#define __ANNF_HPP__ #define __ANNF_HPP__
#include <vector>
#include <stack>
#include <limits>
#include <algorithm>
#include <iterator>
#include <iostream>
#include <fstream>
#include <time.h>
#include <functional>
#include "norm2.hpp" #include "norm2.hpp"
#include "whs.hpp" #include "whs.hpp"
...@@ -95,11 +105,12 @@ public: ...@@ -95,11 +105,12 @@ public:
}; };
template <typename Tp, int cn> int KDTree <Tp, cn>:: template <typename Tp, int cn> int KDTree <Tp, cn>::
getMaxSpreadN(const int _left, const int _right) const getMaxSpreadN(const int left, const int right) const
{ {
cv::Vec<Tp, cn> maxValue = data[ idx[_left] ], cv::Vec<Tp, cn> maxValue = data[ idx[left] ],
minValue = data[ idx[_left] ]; minValue = data[ idx[left] ];
for (int i = _left + 1; i < _right; i += cn)
for (int i = left + 1; i < right; ++i)
for (int j = 0; j < cn; ++j) for (int j = 0; j < cn; ++j)
{ {
minValue[j] = std::min( minValue[j], data[idx[i]][j] ); minValue[j] = std::min( minValue[j], data[idx[i]][j] );
...@@ -143,6 +154,7 @@ KDTree(const cv::Mat &img, const int _leafNumber, const int _zeroThresh) ...@@ -143,6 +154,7 @@ KDTree(const cv::Mat &img, const int _leafNumber, const int _zeroThresh)
} }
int nth = _left + (_right - _left)/2; int nth = _left + (_right - _left)/2;
int dimIdx = getMaxSpreadN(_left, _right); int dimIdx = getMaxSpreadN(_left, _right);
KDTreeComparator comp( this, dimIdx ); KDTreeComparator comp( this, dimIdx );
...@@ -168,8 +180,8 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist) ...@@ -168,8 +180,8 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist)
if (abs(ny - y) < zeroThresh && if (abs(ny - y) < zeroThresh &&
abs(nx - x) < zeroThresh) abs(nx - x) < zeroThresh)
continue; continue;
if (nx > width - 1 || nx < 1 || if (nx >= width - 1 || nx < 1 ||
ny > height - 1 || ny > 1 ) ny >= height - 1 || ny < 1 )
continue; continue;
double ndist = norm2(data[idx0], data[idx[k]]); double ndist = norm2(data[idx0], data[idx[k]]);
...@@ -184,9 +196,12 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist) ...@@ -184,9 +196,12 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist)
/************************** ANNF search **************************/ /************************** ANNF search **************************/
static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &transforms, static void dominantTransforms(const cv::Mat &img, std::vector <cv::Point2i> &transforms,
const int nTransform, const int psize) const int nTransform, const int psize)
{ {
const int zeroThresh = 2*psize;
const int leafNum = 64;
/** Walsh-Hadamard Transformation **/ /** Walsh-Hadamard Transformation **/
std::vector <cv::Mat> channels; std::vector <cv::Mat> channels;
...@@ -204,7 +219,7 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr ...@@ -204,7 +219,7 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
cv::Mat whs; // Walsh-Hadamard series cv::Mat whs; // Walsh-Hadamard series
cv::merge(channels, whs); cv::merge(channels, whs);
KDTree <float, 24> kdTree(whs, 16, 32); KDTree <float, 24> kdTree(whs, leafNum, zeroThresh);
std::vector <int> annf( whs.total(), 0 ); std::vector <int> annf( whs.total(), 0 );
/** Propagation-assisted kd-tree search **/ /** Propagation-assisted kd-tree search **/
...@@ -217,11 +232,11 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr ...@@ -217,11 +232,11 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
int dy[] = {0, 1, 0}, dx[] = {0, 0, 1}; int dy[] = {0, 1, 0}, dx[] = {0, 0, 1};
for (int k = 0; k < int( sizeof(dy)/sizeof(int) ); ++k) for (int k = 0; k < int( sizeof(dy)/sizeof(int) ); ++k)
if (i - dy[k] >= 0 && j - dx[k] >= 0) if ( i - dy[k] >= 0 && j - dx[k] >= 0 )
{ {
int neighbor = (i - dy[k])*whs.cols + (j - dx[k]); int neighbor = (i - dy[k])*whs.cols + (j - dx[k]);
int leafIdx = k == 0 ? neighbor : int leafIdx = (dx[k] == 0 && dy[k] == 0)
annf[neighbor] + dy[k]*whs.cols + dx[k]; ? neighbor : annf[neighbor] + dy[k]*whs.cols + dx[k];
kdTree.updateDist(leafIdx, current, kdTree.updateDist(leafIdx, current,
annf[i*whs.cols + j], dist); annf[i*whs.cols + j], dist);
} }
...@@ -232,8 +247,8 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr ...@@ -232,8 +247,8 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
cv::Mat_<double> annfHist(2*whs.rows - 1, 2*whs.cols - 1, 0.0), cv::Mat_<double> annfHist(2*whs.rows - 1, 2*whs.cols - 1, 0.0),
_annfHist(2*whs.rows - 1, 2*whs.cols - 1, 0.0); _annfHist(2*whs.rows - 1, 2*whs.cols - 1, 0.0);
for (size_t i = 0; i < annf.size(); ++i) for (size_t i = 0; i < annf.size(); ++i)
++annfHist( (annf[i] - int(i))/whs.cols + whs.rows - 1, ++annfHist( annf[i]/whs.cols - int(i)/whs.cols + whs.rows - 1,
(annf[i] - int(i))%whs.cols + whs.cols - 1 ); annf[i]%whs.cols - int(i)%whs.cols + whs.cols - 1 );
cv::GaussianBlur( annfHist, annfHist, cv::GaussianBlur( annfHist, annfHist,
cv::Size(0, 0), std::sqrt(2.0), 0.0, cv::BORDER_CONSTANT); cv::Size(0, 0), std::sqrt(2.0), 0.0, cv::BORDER_CONSTANT);
...@@ -264,9 +279,7 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr ...@@ -264,9 +279,7 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
for (int i = 0; i < nTransform; ++i) for (int i = 0; i < nTransform; ++i)
{ {
int idx = amount[i].second; int idx = amount[i].second;
transforms[i] = cv::Matx33f(1, 0, float(shiftM[idx].x), transforms[i] = cv::Point2i( shiftM[idx].x, shiftM[idx].y );
0, 1, float(shiftM[idx].y),
0, 0, 1 );
} }
} }
......
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __BLENDING_HPP__
#define __BLENDING_HPP__
#endif /* __BLENDING_HPP__ */
\ No newline at end of file
...@@ -254,7 +254,7 @@ TWeight GCGraph<TWeight>::maxFlow() ...@@ -254,7 +254,7 @@ TWeight GCGraph<TWeight>::maxFlow()
minWeight = MIN(minWeight, weight); minWeight = MIN(minWeight, weight);
CV_Assert( minWeight > 0 ); CV_Assert( minWeight > 0 );
} }
weight = fabs(v->weight); weight = abs( TWeight(v->weight) );
minWeight = MIN(minWeight, weight); minWeight = MIN(minWeight, weight);
CV_Assert( minWeight > 0 ); CV_Assert( minWeight > 0 );
} }
......
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
#include <algorithm> #include <algorithm>
#include <iterator> #include <iterator>
#include <iostream> #include <iostream>
#include <fstream>
#include <time.h> #include <time.h>
#include <functional> #include <functional>
...@@ -59,58 +60,227 @@ ...@@ -59,58 +60,227 @@
#include "opencv2/highgui.hpp" #include "opencv2/highgui.hpp"
namespace xphotoInternal #include "photomontage.hpp"
{ #include "annf.hpp"
# include "photomontage.hpp" #include "advanced_types.hpp"
# include "annf.hpp"
}
namespace cv namespace cv
{ {
namespace xphoto namespace xphoto
{ {
template <typename Tp, unsigned int cn> template <typename Tp, unsigned int cn>
static void shiftMapInpaint(const Mat &src, const Mat &mask, Mat &dst, static void shiftMapInpaint( const Mat &_src, const Mat &_mask, Mat &dst,
const int nTransform = 60, const int psize = 8) const int nTransform = 60, const int psize = 8, const cv::Point2i dsize = cv::Point2i(800, 600) )
{ {
/** Preparing input **/ /** Preparing input **/
cv::Mat img; cv::Mat src, mask, img, dmask, ddmask;
const float ls = std::max(/**/ std::min( /*...*/
std::max(_src.rows, _src.cols)/float(dsize.x),
std::min(_src.rows, _src.cols)/float(dsize.y)
), 1.0f /**/);
cv::resize(_mask, mask, _mask.size()/ls, 0, 0, cv::INTER_NEAREST);
cv::resize(_src, src, _src.size()/ls, 0, 0, cv::INTER_AREA);
src.convertTo( img, CV_32F ); src.convertTo( img, CV_32F );
img.setTo(0, 255 - mask); img.setTo(0, 255 - mask);
cv::erode( mask, dmask, cv::Mat(), cv::Point(-1,-1), 2);
cv::erode(dmask, ddmask, cv::Mat(), cv::Point(-1,-1), 2);
std::vector <Point2i> pPath;
cv::Mat_<int> backref( ddmask.size(), int(-1) );
for (int i = 0; i < ddmask.rows; ++i)
{
uchar *dmask_data = (uchar *) ddmask.template ptr<uchar>(i);
int *backref_data = (int *) backref.template ptr< int >(i);
for (int j = 0; j < ddmask.cols; ++j)
if (dmask_data[j] == 0)
{
backref_data[j] = int(pPath.size());
pPath.push_back( cv::Point(j, i) );
}
}
/** ANNF computation **/ /** ANNF computation **/
std::vector <Matx33f> transforms( nTransform ); std::vector <cv::Point2i> transforms( nTransform );
xphotoInternal::dominantTransforms(img, dominantTransforms(img, transforms, nTransform, psize);
transforms, nTransform, psize); transforms.push_back( cv::Point2i(0, 0) );
/** Warping **/ /** Warping **/
std::vector <Mat> images( nTransform + 1 ); // source image transformed with transforms std::vector <std::vector <cv::Vec <float, cn> > > pointSeq( pPath.size() ); // source image transformed with transforms
std::vector <Mat> masks( nTransform + 1 ); // definition domain for current shift std::vector <int> labelSeq( pPath.size() ); // resulting label sequence
std::vector <std::vector <int> > linkIdx( pPath.size() ); // neighbor links for pointSeq elements
std::vector <std::vector <unsigned char > > maskSeq( pPath.size() ); // corresponding mask
Mat_<uchar> invMask = 255 - mask; for (size_t i = 0; i < pPath.size(); ++i)
dilate(invMask, invMask, Mat(), Point(-1,-1), 2); {
uchar xmask = dmask.template at<uchar>(pPath[i]);
img.copyTo( images[0] ); for (int j = 0; j < nTransform + 1; ++j)
mask.copyTo( masks[0] ); {
cv::Point2i u = pPath[i] + transforms[j];
for (int i = 0; i < nTransform; ++i) unsigned char vmask = 0;
cv::Vec <float, cn> vimg = 0;
if ( u.y < src.rows && u.y >= 0
&& u.x < src.cols && u.x >= 0 )
{ {
warpPerspective( images[0], images[i + 1], transforms[i], if ( xmask == 0 || j == nTransform )
images[0].size(), INTER_LINEAR ); vmask = mask.template at<uchar>(u);
vimg = img.template at<cv::Vec<float, cn> >(u);
}
maskSeq[i].push_back(vmask);
pointSeq[i].push_back(vimg);
if (vmask != 0)
labelSeq[i] = j;
}
cv::Point2i p[] = {
pPath[i] + cv::Point2i(0, +1),
pPath[i] + cv::Point2i(+1, 0)
};
warpPerspective( masks[0], masks[i + 1], transforms[i], for (uint j = 0; j < sizeof(p)/sizeof(cv::Point2i); ++j)
masks[0].size(), INTER_NEAREST); if ( p[j].y < src.rows && p[j].y >= 0 &&
masks[i + 1] &= invMask; p[j].x < src.cols && p[j].x >= 0 )
linkIdx[i].push_back( backref(p[j]) );
else
linkIdx[i].push_back( -1 );
} }
/** Stitching **/ /** Stitching **/
Mat photomontageResult; photomontage( pointSeq, maskSeq, linkIdx, labelSeq );
xphotoInternal::Photomontage < cv::Vec <float, cn> >( images, masks )
.assignResImage(photomontageResult); /** Upscaling **/
if (ls != 1)
{
_src.convertTo( img, CV_32F );
std::vector <Point2i> __pPath = pPath; pPath.clear();
cv::Mat_<int> __backref( img.size(), -1 );
std::vector <std::vector <cv::Vec <float, cn> > > __pointSeq = pointSeq; pointSeq.clear();
std::vector <int> __labelSeq = labelSeq; labelSeq.clear();
std::vector <std::vector <int> > __linkIdx = linkIdx; linkIdx.clear();
std::vector <std::vector <unsigned char > > __maskSeq = maskSeq; maskSeq.clear();
for (size_t i = 0; i < __pPath.size(); ++i)
{
cv::Point2i p[] = {
__pPath[i] + cv::Point2i(0, -1),
__pPath[i] + cv::Point2i(-1, 0)
};
for (uint j = 0; j < sizeof(p)/sizeof(cv::Point2i); ++j)
if ( p[j].y < src.rows && p[j].y >= 0 &&
p[j].x < src.cols && p[j].x >= 0 )
__linkIdx[i].push_back( backref(p[j]) );
else
__linkIdx[i].push_back( -1 );
}
for (size_t k = 0; k < __pPath.size(); ++k)
{
int clabel = __labelSeq[k];
int nearSeam = 0;
for (size_t i = 0; i < __linkIdx[k].size(); ++i)
nearSeam |= ( __linkIdx[k][i] == -1
|| clabel != __labelSeq[__linkIdx[k][i]] );
if (nearSeam != 0)
for (int i = 0; i < ls; ++i)
for (int j = 0; j < ls; ++j)
{
cv::Point2i u = ls*(__pPath[k] + transforms[__labelSeq[k]]) + cv::Point2i(j, i);
pPath.push_back( ls*__pPath[k] + cv::Point2i(j, i) );
labelSeq.push_back( 0 );
__backref(i, j) = int( pPath.size() );
cv::Point2i dv[] = {
cv::Point2i(0, 0),
cv::Point2i(-1, 0),
cv::Point2i(+1, 0),
cv::Point2i(0, -1),
cv::Point2i(0, +1)
};
std::vector <cv::Vec <float, cn> > pointVec;
std::vector <uchar> maskVec;
for (uint q = 0; q < sizeof(dv)/sizeof(cv::Point2i); ++q)
if (u.x + dv[q].x >= 0 && u.x + dv[q].x < img.cols
&& u.y + dv[q].y >= 0 && u.y + dv[q].y < img.rows)
{
pointVec.push_back(img.template at<cv::Vec <float, cn> >(u + dv[q]));
maskVec.push_back(_mask.template at<uchar>(u + dv[q]));
}
else
{
pointVec.push_back( cv::Vec <float, cn>::all(0) );
maskVec.push_back( 0 );
}
pointSeq.push_back(pointVec);
maskSeq.push_back(maskVec);
}
else
{
cv::Point2i fromIdx = ls*(__pPath[k] + transforms[__labelSeq[k]]),
toIdx = ls*__pPath[k];
for (int i = 0; i < ls; ++i)
{
cv::Vec <float, cn> *from = img.template ptr<cv::Vec <float, cn> >(fromIdx.y + i) + fromIdx.x;
cv::Vec <float, cn> *to = img.template ptr<cv::Vec <float, cn> >(toIdx.y + i) + toIdx.x;
for (int j = 0; j < ls; ++j)
to[j] = from[j];
}
}
}
for (size_t i = 0; i < pPath.size(); ++i)
{
cv::Point2i p[] = {
pPath[i] + cv::Point2i(0, +1),
pPath[i] + cv::Point2i(+1, 0)
};
std::vector <int> linkVec;
for (uint j = 0; j < sizeof(p)/sizeof(cv::Point2i); ++j)
if ( p[j].y < src.rows && p[j].y >= 0 &&
p[j].x < src.cols && p[j].x >= 0 )
linkVec.push_back( __backref(p[j]) );
else
linkVec.push_back( -1 );
linkIdx.push_back(linkVec);
}
photomontage( pointSeq, maskSeq, linkIdx, labelSeq );
}
/** Writing result **/ /** Writing result **/
photomontageResult.convertTo( dst, dst.type() ); for (size_t i = 0; i < labelSeq.size(); ++i)
{
cv::Vec <float, cn> val = pointSeq[i][labelSeq[i]];
img.template at<cv::Vec <float, cn> >(pPath[i]) = val;
}
img.convertTo( dst, dst.type() );
} }
template <typename Tp, unsigned int cn> template <typename Tp, unsigned int cn>
......
...@@ -40,52 +40,34 @@ ...@@ -40,52 +40,34 @@
#ifndef __NORM2_HPP__ #ifndef __NORM2_HPP__
#define __NORM2_HPP__ #define __NORM2_HPP__
/************************ General template *************************/ template<bool B, class T = void> struct iftype {};
template<class T> struct iftype<true, T> { typedef T type; }; // enable_if
template <typename Tp> static inline Tp sqr(Tp x) { return x*x; } template<class T, T v> struct int_const { // integral_constant
static const T value = v;
typedef T value_type;
typedef int_const type;
operator value_type() const { return value; }
value_type operator()() const { return value; }
};
template <typename Tp, int cn> static inline Tp sqr( cv::Vec<Tp, cn> x) { return x.dot(x); } typedef int_const<bool,true> ttype; // true_type
typedef int_const<bool,false> ftype; // false_type
template <typename Tp> static inline Tp norm2(const Tp &a, const Tp &b) { return sqr(a - b); } template <class T, class U> struct same_as : ftype {};
template <class T> struct same_as<T, T> : ttype {}; // is_same
template <typename Tp, int cn> static inline
Tp norm2(const cv::Vec <Tp, cn> &a, const cv::Vec<Tp, cn> &b) { return sqr(a - b); }
template <typename _Tp> struct is_norm2_type :
int_const<bool, !same_as<_Tp, char>::value
&& !same_as<_Tp, uchar>::value
&& !same_as<_Tp, ushort>::value
&& !same_as<_Tp, uint>::value>{};
template <typename _Tp, int cn> static inline typename iftype< is_norm2_type<_Tp>::value, _Tp >::
type norm2(cv::Vec<_Tp, cn> a, cv::Vec<_Tp, cn> b) { return (a - b).dot(a - b); }
/******************* uchar, char, ushort, uint *********************/ template <typename _Tp> static inline typename iftype< is_norm2_type<_Tp>::value, _Tp >::
type norm2(const _Tp &a, const _Tp &b) { return (a - b)*(a - b); }
static inline int norm2(const uchar &a, const uchar &b) { return sqr(int(a) - int(b)); }
template <int cn> static inline
int norm2(const cv::Vec <uchar, cn> &a, const cv::Vec<uchar, cn> &b)
{
return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
}
static inline int norm2(const char &a, const char &b) { return sqr(int(a) - int(b)); }
template <int cn> static inline
int norm2(const cv::Vec <char, cn> &a, const cv::Vec<char, cn> &b)
{
return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
}
static inline short norm2(const ushort &a, const ushort &b) { return sqr <short>(short(a) - short(b)); }
template <int cn> static inline
short norm2(const cv::Vec <ushort, cn> &a, const cv::Vec<ushort, cn> &b)
{
return sqr( cv::Vec<short, cn>(a) - cv::Vec<short, cn>(b) );
}
static inline int norm2(const uint &a, const uint &b) { return sqr(int(a) - int(b)); }
template <int cn> static inline
int norm2(const cv::Vec <uint, cn> &a, const cv::Vec<uint, cn> &b)
{
return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
}
#endif /* __NORM2_HPP__ */ #endif /* __NORM2_HPP__ */
\ No newline at end of file
...@@ -40,13 +40,33 @@ ...@@ -40,13 +40,33 @@
#ifndef __PHOTOMONTAGE_HPP__ #ifndef __PHOTOMONTAGE_HPP__
#define __PHOTOMONTAGE_HPP__ #define __PHOTOMONTAGE_HPP__
#include <vector>
#include <stack>
#include <limits>
#include <algorithm>
#include <iterator>
#include <iostream>
#include <fstream>
#include <time.h>
#include <functional>
#include "norm2.hpp" #include "norm2.hpp"
#include "blending.hpp"
namespace gcoptimization
{
#include "gcgraph.hpp" #include "gcgraph.hpp"
#define GCInfinity 10*1000*1000*1000.0
typedef float TWeight;
typedef int labelTp;
#define GCInfinity 10*1000*1000
#define eps 0.02 #define eps 0.02
template <typename Tp> static int min_idx(std::vector <Tp> vec) template <typename Tp> static int min_idx(std::vector <Tp> vec)
{ {
return int( std::min_element(vec.begin(), vec.end()) - vec.begin() ); return int( std::min_element(vec.begin(), vec.end()) - vec.begin() );
...@@ -58,22 +78,17 @@ template <typename Tp> static int min_idx(std::vector <Tp> vec) ...@@ -58,22 +78,17 @@ template <typename Tp> static int min_idx(std::vector <Tp> vec)
template <typename Tp> class Photomontage template <typename Tp> class Photomontage
{ {
private: private:
const std::vector <cv::Mat> &images; // vector of images for different labels const std::vector <std::vector <Tp> > &pointSeq; // points for stitching
const std::vector <cv::Mat> &masks; // vector of definition domains for each image const std::vector <std::vector <uchar> > &maskSeq; // corresponding masks
std::vector <cv::Mat> labelings; // vector of labelings for different expansions const std::vector <std::vector <int> > &linkIdx; // vector of neighbors for pointSeq
std::vector <double> distances; // vector of max-flow costs for different labeling
const int height; std::vector <std::vector <labelTp> > labelings; // vector of labelings
const int width; std::vector <TWeight> distances; // vector of max-flow costs for different labeling
const int type;
const int channels;
const int lsize;
cv::Mat x_i; // current best labeling std::vector <labelTp> &labelSeq; // current best labeling
double singleExpansion(const int alpha); // single neighbor computing TWeight singleExpansion(const int alpha); // single neighbor computing
void gradientDescent(); // gradient descent in alpha-expansion topology
class ParallelExpansion : public cv::ParallelLoopBody class ParallelExpansion : public cv::ParallelLoopBody
{ {
...@@ -88,121 +103,86 @@ private: ...@@ -88,121 +103,86 @@ private:
for (int i = range.start; i <= range.end - 1; ++i) for (int i = range.start; i <= range.end - 1; ++i)
main->distances[i] = main->singleExpansion(i); main->distances[i] = main->singleExpansion(i);
} }
}; } parallelExpansion;
void operator =(const Photomontage <Tp>&) const {}; void operator =(const Photomontage <Tp>&) const {};
protected: protected:
virtual double dist(const Tp &l1p1, const Tp &l1p2, const Tp &l2p1, const Tp &l2p2); virtual TWeight dist(const Tp &l1p1, const Tp &l1p2, const Tp &l2p1, const Tp &l2p2);
virtual void setWeights(GCGraph <double> &graph, const cv::Point &pA, virtual void setWeights(GCGraph <TWeight> &graph,
const cv::Point &pB, const int lA, const int lB, const int lX); const int idx1, const int idx2, const int l1, const int l2, const int lx);
public: public:
Photomontage(const std::vector <cv::Mat> &images, const std::vector <cv::Mat> &masks); void gradientDescent(); // gradient descent in alpha-expansion topology
virtual ~Photomontage(){};
void assignLabeling(cv::Mat &img); Photomontage(const std::vector <std::vector <Tp> > &pointSeq,
void assignResImage(cv::Mat &img); const std::vector <std::vector <uchar> > &maskSeq,
const std::vector <std::vector <int> > &linkIdx,
std::vector <labelTp> &labelSeq);
virtual ~Photomontage(){};
}; };
template <typename Tp> inline double Photomontage <Tp>:: template <typename Tp> inline TWeight Photomontage <Tp>::
dist(const Tp &l1p1, const Tp &l1p2, const Tp &l2p1, const Tp &l2p2) dist(const Tp &l1p1, const Tp &l1p2, const Tp &l2p1, const Tp &l2p2)
{ {
return norm2(l1p1, l2p1) + norm2(l1p2, l2p2); return norm2(l1p1, l2p1) + norm2(l1p2, l2p2);
} }
template <typename Tp> void Photomontage <Tp>:: template <typename Tp> void Photomontage <Tp>::
setWeights(GCGraph <double> &graph, const cv::Point &pA, const cv::Point &pB, const int lA, const int lB, const int lX) setWeights(GCGraph <TWeight> &graph, const int idx1, const int idx2,
const int l1, const int l2, const int lx)
{ {
if (lA == lB) if (l1 == l2)
{ {
/** Link from A to B **/ /** Link from A to B **/
double weightAB = dist( images[lA].template at<Tp>(pA), TWeight weightAB = dist( pointSeq[idx1][l1], pointSeq[idx2][l1],
images[lA].template at<Tp>(pB), pointSeq[idx1][lx], pointSeq[idx2][lx] );
images[lX].template at<Tp>(pA), graph.addEdges( idx1, idx2, weightAB, weightAB );
images[lX].template at<Tp>(pB) );
graph.addEdges( int(pA.y*width + pA.x), int(pB.y*width + pB.x), weightAB, weightAB);
} }
else else
{ {
int X = graph.addVtx(); int X = graph.addVtx();
/** Link from X to sink **/ /** Link from X to sink **/
double weightXS = dist( images[lA].template at<Tp>(pA), TWeight weightXS = dist( pointSeq[idx1][l1], pointSeq[idx2][l1],
images[lA].template at<Tp>(pB), pointSeq[idx1][l2], pointSeq[idx2][l2] );
images[lB].template at<Tp>(pA), graph.addTermWeights( X, 0, weightXS );
images[lB].template at<Tp>(pB) );
graph.addTermWeights(X, 0, weightXS);
/** Link from A to X **/ /** Link from A to X **/
double weightAX = dist( images[lA].template at<Tp>(pA), TWeight weightAX = dist( pointSeq[idx1][l1], pointSeq[idx2][l1],
images[lA].template at<Tp>(pB), pointSeq[idx1][lx], pointSeq[idx2][lx] );
images[lX].template at<Tp>(pA), graph.addEdges( idx1, X, weightAX, weightAX );
images[lX].template at<Tp>(pB) );
graph.addEdges( int(pA.y*width + pA.x), X, weightAX, weightAX);
/** Link from X to B **/ /** Link from X to B **/
double weightXB = dist( images[lX].template at<Tp>(pA), TWeight weightXB = dist( pointSeq[idx1][lx], pointSeq[idx1][lx],
images[lX].template at<Tp>(pB), pointSeq[idx1][l2], pointSeq[idx1][l2] );
images[lB].template at<Tp>(pA), graph.addEdges( X, idx2, weightXB, weightXB );
images[lB].template at<Tp>(pB) );
graph.addEdges(X, int(pB.y*width + pB.x), weightXB, weightXB);
} }
} }
template <typename Tp> double Photomontage <Tp>:: template <typename Tp> TWeight Photomontage <Tp>::
singleExpansion(const int alpha) singleExpansion(const int alpha)
{ {
int actualEdges = (height - 1)*width + height*(width - 1); GCGraph <TWeight> graph( 3*int(pointSeq.size()), 4*int(pointSeq.size()) );
GCGraph <double> graph(actualEdges + height*width, 2*actualEdges);
/** Terminal links **/ /** Terminal links **/
for (int i = 0; i < height; ++i) for (size_t i = 0; i < maskSeq.size(); ++i)
{
const uchar *maskAlphaRow = masks[alpha].template ptr <uchar>(i);
const int *labelRow = (const int *) x_i.template ptr <int>(i);
for (int j = 0; j < width; ++j)
graph.addTermWeights( graph.addVtx(), graph.addTermWeights( graph.addVtx(),
maskAlphaRow[j] ? 0 : GCInfinity, maskSeq[i][alpha] ? TWeight(0) : TWeight(GCInfinity), 0 );
masks[ labelRow[j] ].template at<uchar>(i, j) ? 0 : GCInfinity );
}
/** Neighbor links **/ /** Neighbor links **/
for (int i = 0; i < height - 1; ++i) for (size_t i = 0; i < pointSeq.size(); ++i)
{ for (size_t j = 0; j < linkIdx[i].size(); ++j)
const int *currentRow = (const int *) x_i.template ptr <int>(i); if ( linkIdx[i][j] != -1)
const int *nextRow = (const int *) x_i.template ptr <int>(i + 1); setWeights( graph, int(i), linkIdx[i][j],
labelSeq[i], labelSeq[linkIdx[i][j]], alpha );
for (int j = 0; j < width - 1; ++j)
{
setWeights( graph, cv::Point(j, i), cv::Point(j + 1, i), currentRow[j], currentRow[j + 1], alpha );
setWeights( graph, cv::Point(j, i), cv::Point(j, i + 1), currentRow[j], nextRow[j], alpha );
}
setWeights( graph, cv::Point(width - 1, i), cv::Point(width - 1, i + 1),
currentRow[width - 1], nextRow[width - 1], alpha );
}
const int *currentRow = (const int *) x_i.template ptr <int>(height - 1);
for (int i = 0; i < width - 1; ++i)
setWeights( graph, cv::Point(i, height - 1), cv::Point(i + 1, height - 1),
currentRow[i], currentRow[i + 1], alpha );
/** Max-flow computation **/ /** Max-flow computation **/
double result = graph.maxFlow(); TWeight result = graph.maxFlow();
/** Writing results **/ /** Writing results **/
labelings[alpha].create( height, width, CV_32SC1 ); for (size_t i = 0; i < pointSeq.size(); ++i)
for (int i = 0; i < height; ++i) labelings[i][alpha] = graph.inSourceSegment(int(i)) ? labelSeq[i] : alpha;
{
const int *inRow = (const int *) x_i.template ptr <int>(i);
int *outRow = (int *) labelings[alpha].template ptr <int>(i);
for (int j = 0; j < width; ++j)
outRow[j] = graph.inSourceSegment(i*width + j)
? inRow[j] : alpha;
}
return result; return result;
} }
...@@ -210,55 +190,51 @@ singleExpansion(const int alpha) ...@@ -210,55 +190,51 @@ singleExpansion(const int alpha)
template <typename Tp> void Photomontage <Tp>:: template <typename Tp> void Photomontage <Tp>::
gradientDescent() gradientDescent()
{ {
double optValue = std::numeric_limits<double>::max(); TWeight optValue = std::numeric_limits<TWeight>::max();
for (int num = -1; /**/; num = -1) for (int num = -1; /**/; num = -1)
{ {
parallel_for_( cv::Range(0, lsize), int range = int( pointSeq[0].size() );
ParallelExpansion(this) ); parallel_for_( cv::Range(0, range), parallelExpansion );
int minIndex = min_idx(distances); int minIndex = min_idx(distances);
double minValue = distances[minIndex]; TWeight minValue = distances[minIndex];
if (minValue < (1.00 - eps)*optValue) if (minValue < (1.00 - eps)*optValue)
optValue = distances[num = minIndex]; optValue = distances[num = minIndex];
if (num == -1) if (num == -1)
break; break;
labelings[num].copyTo(x_i);
for (size_t i = 0; i < labelSeq.size(); ++i)
labelSeq[i] = labelings[i][num];
} }
} }
template <typename Tp> void Photomontage <Tp>:: template <typename Tp> Photomontage <Tp>::
assignLabeling(cv::Mat &img) Photomontage( const std::vector <std::vector <Tp> > &_pointSeq,
const std::vector <std::vector <uchar> > &_maskSeq,
const std::vector <std::vector <int> > &_linkIdx,
std::vector <labelTp> &_labelSeq )
:
pointSeq(_pointSeq), maskSeq(_maskSeq), linkIdx(_linkIdx),
distances(pointSeq[0].size()), labelSeq(_labelSeq), parallelExpansion(this)
{ {
x_i.setTo(0); size_t lsize = pointSeq[0].size();
gradientDescent(); labelings.assign( pointSeq.size(),
x_i.copyTo(img); std::vector <labelTp>( lsize ) );
} }
template <typename Tp> void Photomontage <Tp>::
assignResImage(cv::Mat &img)
{
cv::Mat optimalLabeling;
assignLabeling(optimalLabeling);
img.create( height, width, type );
for (int i = 0; i < height; ++i)
for (int j = 0; j < width; ++j)
{
cv::Mat M = images[optimalLabeling.template at<int>(i, j)];
img.template at<Tp>(i, j) = M.template at<Tp>(i, j);
}
} }
template <typename Tp> Photomontage <Tp>:: template <typename Tp> static inline
Photomontage(const std::vector <cv::Mat> &_images, const std::vector <cv::Mat> &_masks) void photomontage( const std::vector <std::vector <Tp> > &pointSeq,
: const std::vector <std::vector <uchar> > &maskSeq,
images(_images), masks(_masks), labelings(images.size()), distances(images.size()), const std::vector <std::vector <int> > &linkIdx,
height(int(images[0].rows)), width(int(images[0].cols)), type(images[0].type()), std::vector <gcoptimization::labelTp> &labelSeq )
channels(images[0].channels()), lsize(int(images.size())), x_i(height, width, CV_32SC1){} {
gcoptimization::Photomontage <Tp>(pointSeq, maskSeq,
linkIdx, labelSeq).gradientDescent();
}
#endif /* __PHOTOMONTAGE_HPP__ */ #endif /* __PHOTOMONTAGE_HPP__ */
...@@ -137,8 +137,11 @@ static void rgb2whs(const cv::Mat &src, cv::Mat &dst, const int nProjections, co ...@@ -137,8 +137,11 @@ static void rgb2whs(const cv::Mat &src, cv::Mat &dst, const int nProjections, co
nextProjection(projections, snake_idx[i - 1], nextProjection(projections, snake_idx[i - 1],
snake_idx[i], npsize); snake_idx[i], npsize);
int pad = 0;
cv::merge(projections, img); cv::merge(projections, img);
img(cv::Rect(npsize, npsize, src.cols, src.rows)).copyTo(dst); img(cv::Rect(npsize + pad, npsize + pad, src.cols - pad,
src.rows - pad)).copyTo(dst);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment