images -> vectors in xphoto/inpainting

modules/xphoto/samples/inpainting.cpp

@@ -6,6 +6,8 @@
 #include "opencv2/core/utility.hpp"
 #include "opencv2/imgproc/types_c.h"
 
+#include <ctime>
+#include <iostream>
 
 const char* keys =
 {
@@ -54,9 +56,15 @@ int main( int argc, const char** argv )
         printf( "Cannot read image file: %s\n", maskFilename.c_str() );
         return -1;
     }
+    cv::threshold(mask, mask, 128, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
 
     cv::Mat res(src.size(), src.type());
+
+    int time = clock();
     cv::xphoto::inpaint( src, mask, res, cv::xphoto::INPAINT_SHIFTMAP );
+    std::cout << "time = " << (clock() - time)
+        / double(CLOCKS_PER_SEC) << std::endl;
+
     cv::cvtColor(res, res, CV_Lab2RGB);
 
     if ( outFilename == "" )

modules/xphoto/src/advanced_types.hpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of Intel Corporation may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifndef __ADVANCED_TYPES_HPP__
+#define __ADVANCED_TYPES_HPP__
+#ifdef __cplusplus
+
+#include <opencv2/core.hpp>
+
+/********************* Functions *********************/
+
+namespace cv
+{
+
+template <typename _Tp, typename _Tp2> static inline
+    cv::Size_<_Tp> operator * (const _Tp2 x, const cv::Size_<_Tp> &sz)
+{
+    return cv::Size_<_Tp>(cv::saturate_cast<_Tp>(x*sz.width), cv::saturate_cast<_Tp>(x*sz.height));
+}
+
+template <typename _Tp, typename _Tp2> static inline
+    cv::Size_<_Tp> operator / (const cv::Size_<_Tp> &sz, const _Tp2 x)
+{
+    return cv::Size_<_Tp>(cv::saturate_cast<_Tp>(sz.width/x), cv::saturate_cast<_Tp>(sz.height/x));
+}
+
+} // cv
+
+#endif
+#endif /* __ADVANCED_TYPES_HPP__ */
\ No newline at end of file

modules/xphoto/src/annf.hpp

@@ -40,6 +40,16 @@
 #ifndef __ANNF_HPP__
 #define __ANNF_HPP__
 
+#include <vector>
+#include <stack>
+#include <limits>
+#include <algorithm>
+#include <iterator>
+#include <iostream>
+#include <fstream>
+#include <time.h>
+#include <functional>
+
 #include "norm2.hpp"
 #include "whs.hpp"
 
@@ -170,8 +180,8 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist)
         if (abs(ny - y) < zeroThresh &&
             abs(nx - x) < zeroThresh)
             continue;
-        if (nx > width  - 1 || nx < 1 ||
-            ny > height - 1 || ny < 1 )
+        if (nx >= width  - 1 || nx < 1 ||
+            ny >= height - 1 || ny < 1 )
             continue;
 
         double ndist = norm2(data[idx0], data[idx[k]]);
@@ -186,11 +196,11 @@ updateDist(const int leaf, const int &idx0, int &bestIdx, double &dist)
 
 /************************** ANNF search **************************/
 
-static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &transforms,
+static void dominantTransforms(const cv::Mat &img, std::vector <cv::Point2i> &transforms,
                                const int nTransform, const int psize)
 {
     const int zeroThresh = 2*psize;
-    const int leafNum = 63;
+    const int leafNum = 64;
 
     /** Walsh-Hadamard Transformation **/
 
@@ -222,13 +232,13 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
 
             int dy[] = {0, 1, 0}, dx[] = {0, 0, 1};
             for (int k = 0; k < int( sizeof(dy)/sizeof(int) ); ++k)
-                if (i - dy[k] >= 0 && j - dx[k] >= 0)
+                if ( i - dy[k] >= 0 && j - dx[k] >= 0 )
                 {
                     int neighbor = (i - dy[k])*whs.cols + (j - dx[k]);
                     int leafIdx = (dx[k] == 0 && dy[k] == 0)
                         ? neighbor : annf[neighbor] + dy[k]*whs.cols + dx[k];
                     kdTree.updateDist(leafIdx, current,
-                               annf[i*whs.cols + j], dist);
+                                annf[i*whs.cols + j], dist);
                 }
         }
 
@@ -265,15 +275,11 @@ static void dominantTransforms(const cv::Mat &img, std::vector <cv::Matx33f> &tr
     std::partial_sort( amount.begin(), amount.begin() + nTransform,
         amount.end(), std::greater< std::pair<double, int> >() );
 
-    std::ofstream out("C:/Users/Yury/Projects/inpaint/output/log.log");
     transforms.resize(nTransform);
     for (int i = 0; i < nTransform; ++i)
     {
         int idx = amount[i].second;
-        transforms[i] = cv::Matx33f(1, 0, float(shiftM[idx].x),
-                                    0, 1, float(shiftM[idx].y),
-                                    0, 0,         1          );
-        out << int(shiftM[idx].x) << "," << int(shiftM[idx].y) << std::endl;
+        transforms[i] = cv::Point2i( shiftM[idx].x, shiftM[idx].y );
     }
 }
 

modules/xphoto/src/blending.hpp

+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of Intel Corporation may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifndef __BLENDING_HPP__
+#define __BLENDING_HPP__
+
+
+
+#endif /* __BLENDING_HPP__ */
\ No newline at end of file

modules/xphoto/src/gcgraph.hpp

@@ -254,7 +254,7 @@ TWeight GCGraph<TWeight>::maxFlow()
                 minWeight = MIN(minWeight, weight);
                 CV_Assert( minWeight > 0 );
             }
-            weight = fabs(v->weight);
+            weight = abs( TWeight(v->weight) );
             minWeight = MIN(minWeight, weight);
             CV_Assert( minWeight > 0 );
         }

modules/xphoto/src/inpainting.cpp

@@ -60,59 +60,137 @@
 
 #include "opencv2/highgui.hpp"
 
-namespace xphotoInternal
-{
-#  include "photomontage.hpp"
-#  include "annf.hpp"
-}
+#include "photomontage.hpp"
+#include "annf.hpp"
+#include "advanced_types.hpp"
 
 namespace cv
 {
 namespace xphoto
 {
-
     template <typename Tp, unsigned int cn>
-    static void shiftMapInpaint(const Mat &src, const Mat &mask, Mat &dst,
-        const int nTransform = 60, const int psize = 8)
+    static void shiftMapInpaint( const Mat &_src, const Mat &_mask, Mat &dst,
+        const int nTransform = 60, const int psize = 8, const cv::Point2i dsize = cv::Point2i(800, 600) )
     {
         /** Preparing input **/
-        cv::Mat img;
+        cv::Mat src, mask, img, dmask, ddmask;
+
+        const float ls = std::max(/**/ std::min( /*...*/
+            std::max(_src.rows, _src.cols)/float(dsize.x),
+            std::min(_src.rows, _src.cols)/float(dsize.y)
+                                               ), 1.0f /**/);
+
+
+        cv::resize(_mask, mask, _mask.size()/ls, 0, 0, cv::INTER_NEAREST);
+        cv::resize(_src,  src,  _src.size()/ls,  0, 0,    cv::INTER_AREA);
+
         src.convertTo( img, CV_32F );
         img.setTo(0, 255 - mask);
 
+        cv::erode( mask,  dmask, cv::Mat(), cv::Point(-1,-1), 2);
+        cv::erode(dmask, ddmask, cv::Mat(), cv::Point(-1,-1), 2);
+
+        std::vector <Point2i> pPath;
+        cv::Mat_<int> backref( ddmask.size(), int(-1) );
+
+        for (int i = 0; i < ddmask.rows; ++i)
+        {
+            uchar *dmask_data = (uchar *) ddmask.template ptr<uchar>(i);
+            int *backref_data = (int *) backref.template ptr< int >(i);
+
+            for (int j = 0; j < ddmask.cols; ++j)
+                if (dmask_data[j] == 0)
+                {
+                    backref_data[j] = int(pPath.size());
+                     pPath.push_back( cv::Point(j, i) );
+                }
+        }
+
         /** ANNF computation **/
-        std::vector <Matx33f> transforms( nTransform );
-        xphotoInternal::dominantTransforms(img,
-                    transforms, nTransform, psize);
+        std::vector <cv::Point2i> transforms( nTransform );
+        dominantTransforms(img, transforms, nTransform, psize);
+        transforms.push_back( cv::Point2i(0, 0) );
 
         /** Warping **/
-        std::vector <Mat> images( nTransform + 1 ); // source image transformed with transforms
-        std::vector <Mat> masks( nTransform + 1 );  // definition domain for current shift
+        std::vector <std::vector <cv::Vec <float, cn> > > pointSeq( pPath.size() ); // source image transformed with transforms
+        std::vector <int> labelSeq( pPath.size() );                                 // resulting label sequence
+        std::vector <std::vector <int> >  linkIdx( pPath.size() );                  // neighbor links for pointSeq elements
+        std::vector <std::vector <unsigned char > > maskSeq( pPath.size() );        // corresponding mask
 
-        Mat_<uchar> invMask = 255 - mask;
-        dilate(invMask, invMask, Mat(), Point(-1,-1), 2);
+        for (size_t i = 0; i < pPath.size(); ++i)
+        {
+            for (int j = 0; j < nTransform + 1; ++j)
+            {
+                cv::Point2i u = pPath[i] + transforms[j];
+                uchar xmask = dmask.template at<uchar>(pPath[i]);
 
-        img.copyTo( images[0] );
-        mask.copyTo( masks[0] );
+                unsigned char vmask = 0;
+                cv::Vec <float, cn> vimg = 0;
 
-        for (int i = 0; i < nTransform; ++i)
-        {
-            warpPerspective( images[0], images[i + 1], transforms[i],
-                             images[0].size(), INTER_LINEAR );
+                if ( u.y < src.rows && u.y >= 0
+                &&   u.x < src.cols && u.x >= 0 )
+                {
+                    if ( xmask == 0 || j == nTransform )
+                        vmask = mask.template at<uchar>(u);
+                    vimg = img.template at<cv::Vec<float, cn> >(u);
+                }
+
+                maskSeq[i].push_back(vmask);
+                pointSeq[i].push_back(vimg);
+
+                if (vmask != 0)
+                    labelSeq[i] = j;
+            }
+
+            cv::Point2i  p[] = {
+                                 pPath[i] + cv::Point2i(0, +1),
+                                 pPath[i] + cv::Point2i(+1, 0)
+                               };
 
-            warpPerspective( masks[0], masks[i + 1], transforms[i],
-                             masks[0].size(), INTER_NEAREST);
-            cv::imwrite(cv::format("C:/Users/Yury/Projects/inpaint/output/%d.png", i), images[i]);
-            masks[i + 1] &= invMask;
+            for (int j = 0; j < sizeof(p)/sizeof(cv::Point2i); ++j)
+                if ( p[j].y < src.rows && p[j].y >= 0 &&
+                     p[j].x < src.cols && p[j].x >= 0 )
+                    linkIdx[i].push_back( backref(p[j]) );
+                else
+                    linkIdx[i].push_back( -1 );
         }
 
         /** Stitching **/
-        Mat photomontageResult;
-        xphotoInternal::Photomontage < cv::Vec <float, cn> >( images, masks )
-            .assignResImage(photomontageResult);
+        photomontage( pointSeq, maskSeq, linkIdx, labelSeq );
+
+        /** Upscaling **/
+        if (ls != 1)
+        {
+        //    _src.convertTo( img, CV_32F );
+        //
+        //    for (size_t k = 0; k < pPath.size(); ++k)
+        //    {
+        //        int clabel = labelSeq[k];
+        //        int nearSeam = 0;
+        //
+        //        for (size_t i = 0; i < linkIdx[k].size(); ++i)
+        //            nearSeam |= linkIdx[k] == -1
+        //                || clabel != labelSeq[linkIdx[k][i]];
+        //
+        //        if (nearSeam == 0)
+        //            ...
+        //        else
+        //        {
+        //            ...
+        //        }
+        //    }
+        //
+        //    xphotoInternal::Photomontage < cv::Vec <float, cn> > ( /*...*/
+        //        pointSeq, maskSeq, linkIdx, labelSeq );
+        }
 
         /** Writing result **/
-        photomontageResult.convertTo( dst, dst.type() );
+        for (size_t i = 0; i < labelSeq.size(); ++i)
+        {
+            cv::Vec <float, cn> val = pointSeq[i][labelSeq[i]];
+            img.template at<cv::Vec <float, cn> >(pPath[i]) = val;
+        }
+        img.convertTo( dst, dst.type() );
     }
 
     template <typename Tp, unsigned int cn>

modules/xphoto/src/norm2.hpp

@@ -40,52 +40,13 @@
 #ifndef __NORM2_HPP__
 #define __NORM2_HPP__
 
-/************************ General template *************************/
+template <typename _Tp> struct is_norm2_type :
+    std::integral_constant<bool, !std::is_unsigned<_Tp>::value && !std::is_same<_Tp, char>::value> {};
 
-template <typename Tp> static inline Tp sqr(Tp x) { return x*x; }
-
-template <typename Tp, int cn> static inline Tp sqr( cv::Vec<Tp, cn> x) { return x.dot(x); }
-
-template <typename Tp> static inline Tp norm2(const Tp &a, const Tp &b) { return sqr(a - b); }
-
-template <typename Tp, int cn> static inline
-Tp norm2(const cv::Vec <Tp, cn> &a, const cv::Vec<Tp, cn> &b) { return sqr(a - b); }
-
-
-
-/******************* uchar, char, ushort, uint *********************/
-
-static inline int norm2(const uchar &a, const uchar &b) { return sqr(int(a) - int(b)); }
-
-template <int cn> static inline
-    int norm2(const cv::Vec <uchar, cn> &a, const cv::Vec<uchar, cn> &b)
-{
-    return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
-}
-
-static inline int norm2(const char &a, const char &b) { return sqr(int(a) - int(b)); }
-
-template <int cn> static inline
-    int norm2(const cv::Vec <char, cn> &a, const cv::Vec<char, cn> &b)
-{
-    return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
-}
-
-static inline short norm2(const ushort &a, const ushort &b) { return sqr <short>(short(a) - short(b)); }
-
-template <int cn> static inline
-    short norm2(const cv::Vec <ushort, cn> &a, const cv::Vec<ushort, cn> &b)
-{
-    return sqr( cv::Vec<short, cn>(a) - cv::Vec<short, cn>(b) );
-}
-
-static inline int norm2(const uint &a, const uint &b) { return sqr(int(a) - int(b)); }
-
-template <int cn> static inline
-    int norm2(const cv::Vec <uint, cn> &a, const cv::Vec<uint, cn> &b)
-{
-    return sqr( cv::Vec<int, cn>(a) - cv::Vec<int, cn>(b) );
-}
+template <typename _Tp, int cn> static inline typename std::enable_if< is_norm2_type<_Tp>::value, _Tp >::
+    type norm2(cv::Vec<_Tp, cn> a, cv::Vec<_Tp, cn> b) { return (a - b).dot(a - b); }
 
+template <typename _Tp> static inline typename std::enable_if< is_norm2_type<_Tp>::value, _Tp >::
+    type norm2(const _Tp &a, const _Tp &b) { return (a - b)*(a - b); }
 
 #endif /* __NORM2_HPP__ */
\ No newline at end of file