Merge pull request #2296 from si40wiga:fsr-inpaint

* new algorithm Rapid Frequency Selective Reconstruction (FSR) added

* fix compiler warning

* applied changes suggested in alalek's review

* fix trailing whitespace

* xphoto: update inpaint() test

* fix pre-processing of error mask

* xphoto: move inpainting FSR algorithm into a separate file

* xphoto: cleanup inpaining documentation

* xphoto: inpainting fsr - avoid uninitialized values

modules/xphoto/doc/xphoto.bib

@@ -29,3 +29,49 @@
   publisher = {ACM},
   url = {https://www.researchgate.net/profile/Julie_Dorsey/publication/220184746_Fast_Bilateral_Filtering_for_the_Display_of_High_-_dynamic_-_range_Images/links/54566b000cf26d5090a95f96/Fast-Bilateral-Filtering-for-the-Display-of-High-dynamic-range-Images.pdf}
 }
+
+@INPROCEEDINGS{GenserPCS2018,
+   author={N. {Genser} and J. {Seiler} and F. {Schilling} and A. {Kaup}},
+   booktitle={Proc. Picture Coding Symposium (PCS)},
+   title={Signal and Loss Geometry Aware Frequency Selective Extrapolation for Error Concealment},
+   year={2018},
+   pages={159-163},
+   keywords={extrapolation;image reconstruction;video coding;loss geometry aware frequency selective extrapolation;error concealment;complex models;moderate computational complexity;Full HD image;error pattern;adjacent samples;undistorted samples;reconstruction parameters;processing order;High Efficiency Video Coding;content based partitioning;signal characteristics;block based frequency selective extrapolation;Image reconstruction;Extrapolation;Geometry;Partitioning algorithms;Task analysis;Computational modeling;Standards},
+   doi={10.1109/PCS.2018.8456259},
+   month={June},
+}
+
+@ARTICLE{SeilerTIP2015,
+   author={J. {Seiler} and M. {Jonscher} and M. {Schöberl} and A. {Kaup}},
+   journal={IEEE Transactions on Image Processing},
+   title={Resampling Images to a Regular Grid From a Non-Regular Subset of Pixel Positions Using Frequency Selective Reconstruction},
+   year={2015},
+   volume={24},
+   number={11},
+   pages={4540-4555},
+   keywords={Fourier transforms;image reconstruction;resampling images;regular grid;nonregular subset;pixel positions;frequency selective reconstruction;displaying image signals;image signal reconstruction algorithm;Fourier domain;optical transfer function;visual quality;peak signal-to-noise ratio;Image reconstruction;Signal processing algorithms;Reconstruction algorithms;Signal processing;Spatial resolution;;Image reconstruction;non-regular sampling;interpolation},
+   doi={10.1109/TIP.2015.2463084},
+   month={Nov},
+}
+
+@INPROCEEDINGS{GroscheICIP2018,
+   author={S. {Grosche} and J. {Seiler} and A. {Kaup}},
+   booktitle={Proc. 25th IEEE International Conference on Image Processing (ICIP)},
+   title={Iterative Optimization of Quarter Sampling Masks for Non-Regular Sampling Sensors},
+   year={2018},
+   pages={26-30},
+   keywords={extrapolation;image enhancement;image reconstruction;image resolution;image sampling;image sensors;interpolation;iterative methods;optimisation;regression analysis;iterative optimization;nonregular sampling sensors;iterative algorithm;arbitrary quarter sampling mask;reconstruction algorithms;random quarter sampling mask;optimized mask;frequency selective extrapolation;steering kernel regression;nearest neighbor interpolation;linear interpolation;regular imaging sensor;reconstruction quality;noise figure 0.31 dB to 0.68 dB;Image resolution;Image reconstruction;Sensors;Optimization;Energy resolution;Reconstruction algorithms;Image sensors;Non-Regular Sampling;Image reconstruction},
+   doi={10.1109/ICIP.2018.8451658},
+   month={Oct},
+}
+
+@INPROCEEDINGS{GroscheIST2018,
+   author={S. {Grosche} and J. {Seiler} and A. {Kaup}},
+   booktitle={Proc. IEEE International Conference on Imaging Systems and Techniques (IST)},
+   title={Design Techniques for Incremental Non-Regular Image Sampling Patterns},
+   year={2018},
+   pages={1-6},
+   keywords={image reconstruction;image resolution;image sampling;design techniques;incremental nonregular image sampling patterns;image signals;regular two dimensional grid;nonregular sampling patterns;sampling positions;random patterns;regular patterns;arbitrary sampling densities;incremental sampling patterns;sampling density;Image reconstruction;Scanning electron microscopy;Probability distribution;Atomic force microscopy;Reconstruction algorithms;Measurement by laser beam;Image Reconstruction;non-Regular Sampling},
+   doi={10.1109/IST.2018.8577090},
+   month={Oct},
+}

modules/xphoto/include/opencv2/xphoto/inpainting.hpp

@@ -9,9 +9,14 @@
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
+//                       (3-clause BSD License)
 //
-// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2000-2019, Intel Corporation, all rights reserved.
 // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.
+// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+// Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.
+// Copyright (C) 2015-2016, Itseez Inc., all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
@@ -24,8 +29,9 @@
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
-//   * The name of the copyright holders may not be used to endorse or promote products
-//     derived from this software without specific prior written permission.
+//   * Neither the names of the copyright holders nor the names of the contributors
+//    may be used to endorse or promote products derived from this software
+//    without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
@@ -58,24 +64,48 @@ namespace xphoto
 //! @addtogroup xphoto
 //! @{
 
-    //! various inpainting algorithms
+    //! @brief Various inpainting algorithms
+    //! @sa inpaint
     enum InpaintTypes
     {
         /** This algorithm searches for dominant correspondences (transformations) of
         image patches and tries to seamlessly fill-in the area to be inpainted using this
         transformations */
-        INPAINT_SHIFTMAP = 0
+        INPAINT_SHIFTMAP = 0,
+        /** Performs Frequency Selective Reconstruction (FSR).
+        One of the two quality profiles BEST and FAST can be chosen, depending on the time available for reconstruction.
+        See @cite GenserPCS2018 and @cite SeilerTIP2015 for details.
+
+        The algorithm may be utilized for the following areas of application:
+        1. %Error Concealment (Inpainting).
+           The sampling mask indicates the missing pixels of the distorted input
+           image to be reconstructed.
+        2. Non-Regular Sampling.
+           For more information on how to choose a good sampling mask, please review
+           @cite GroscheICIP2018 and @cite GroscheIST2018.
+
+        1-channel grayscale or 3-channel BGR image are accepted.
+
+        Conventional accepted ranges:
+        - 0-255 for CV_8U
+        - 0-65535 for CV_16U
+        - 0-1 for CV_32F/CV_64F.
+        */
+        INPAINT_FSR_BEST = 1,
+        INPAINT_FSR_FAST = 2                     //!< See #INPAINT_FSR_BEST
     };
 
     /** @brief The function implements different single-image inpainting algorithms.
 
-    See the original paper @cite He2012 for details.
+    See the original papers @cite He2012 (Shiftmap) or @cite GenserPCS2018 and @cite SeilerTIP2015 (FSR) for details.
 
-    @param src source image, it could be of any type and any number of channels from 1 to 4. In case of
+    @param src source image
+    - #INPAINT_SHIFTMAP: it could be of any type and any number of channels from 1 to 4. In case of
     3- and 4-channels images the function expect them in CIELab colorspace or similar one, where first
     color component shows intensity, while second and third shows colors. Nonetheless you can try any
     colorspaces.
-    @param mask mask (CV_8UC1), where non-zero pixels indicate valid image area, while zero pixels
+    - #INPAINT_FSR_BEST or #INPAINT_FSR_FAST: 1-channel grayscale or 3-channel BGR image.
+    @param mask mask (#CV_8UC1), where non-zero pixels indicate valid image area, while zero pixels
     indicate area to be inpainted
     @param dst destination image
     @param algorithmType see xphoto::InpaintTypes

modules/xphoto/src/inpainting.cpp

@@ -9,11 +9,19 @@
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
+//                       (3-clause BSD License)
 //
-// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2000-2019, Intel Corporation, all rights reserved.
 // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.
+// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+// Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.
+// Copyright (C) 2015-2016, Itseez Inc., all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
@@ -21,8 +29,9 @@
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
-//   * The name of Intel Corporation may not be used to endorse or promote products
-//     derived from this software without specific prior written permission.
+//   * Neither the names of the copyright holders nor the names of the contributors
+//    may be used to endorse or promote products derived from this software
+//    without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
@@ -46,6 +55,8 @@
 #include <fstream>
 #include <time.h>
 #include <functional>
+#include <string>
+#include <tuple>
 
 #include "opencv2/xphoto.hpp"
 
@@ -62,6 +73,8 @@
 #include "annf.hpp"
 #include "advanced_types.hpp"
 
+#include "inpainting_fsr.impl.hpp"
+
 namespace cv
 {
 namespace xphoto
@@ -298,17 +311,9 @@ namespace xphoto
         }
     }
 
-    /*! The function reconstructs the selected image area from known area.
-    *  \param src : source image.
-    *  \param mask : inpainting mask, 8-bit 1-channel image. Zero pixels indicate the area that needs to be inpainted.
-    *  \param dst : destination image.
-    *  \param algorithmType : inpainting method.
-    */
-    void inpaint(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)
+    static
+    void inpaint_shiftmap(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)
     {
-        CV_Assert( mask.channels() == 1 && mask.depth() == CV_8U );
-        CV_Assert( src.rows == mask.rows && src.cols == mask.cols );
-
         switch ( src.type() )
         {
             case CV_8SC1:
@@ -399,8 +404,25 @@ namespace xphoto
                 CV_Error_( CV_StsNotImplemented,
                     ("Unsupported source image format (=%d)",
                     src.type()) );
-                break;
         }
     }
+
+void inpaint(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)
+{
+    CV_Assert(!src.empty());
+    CV_Assert(!mask.empty());
+    CV_CheckTypeEQ(mask.type(), CV_8UC1, "");
+    CV_Assert(src.rows == mask.rows && src.cols == mask.cols);
+
+    switch (algorithmType)
+    {
+        case xphoto::INPAINT_SHIFTMAP:
+            return inpaint_shiftmap(src, mask, dst, algorithmType);
+        case xphoto::INPAINT_FSR_BEST:
+        case xphoto::INPAINT_FSR_FAST:
+            return inpaint_fsr(src, mask, dst, algorithmType);
+    }
+    CV_Error_(Error::StsNotImplemented, ("Unsupported inpainting algorithm type (=%d)", algorithmType));
 }
-}
+
+}}  // namespace

modules/xphoto/test/test_inpainting.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+#include "test_precomp.hpp"
+
+namespace opencv_test { namespace {
+using namespace xphoto;
+
+
+static void test_inpainting(const Size inputSize, InpaintTypes mode, double expected_psnr, ImreadModes inputMode = IMREAD_COLOR)
+{
+    string original_path = cvtest::findDataFile("cv/shared/lena.png");
+    string mask_path = cvtest::findDataFile("cv/inpaint/mask.png");
+
+    Mat original_ = imread(original_path, inputMode);
+    ASSERT_FALSE(original_.empty()) << "Could not load input image " << original_path;
+
+    Mat mask_ = imread(mask_path, IMREAD_GRAYSCALE);
+    ASSERT_FALSE(mask_.empty()) << "Could not load error mask " << mask_path;
+
+    Mat original, mask;
+    resize(original_, original, inputSize, 0.0, 0.0, INTER_AREA);
+    resize(mask_, mask, inputSize, 0.0, 0.0, INTER_NEAREST);
+
+    Mat mask_valid = (mask == 0);
+    Mat im_distorted(inputSize, original.type(), Scalar::all(0));
+    original.copyTo(im_distorted, mask_valid);
+
+    Mat reconstructed;
+    xphoto::inpaint(im_distorted, mask_valid, reconstructed, mode);
+
+    double adiff_psnr = cvtest::PSNR(original, reconstructed);
+    EXPECT_LE(expected_psnr, adiff_psnr);
+
+#if 0
+    imshow("original", original);
+    imshow("im_distorted", im_distorted);
+    imshow("reconstructed", reconstructed);
+    std::cout << "adiff_psnr=" << adiff_psnr << std::endl;
+    waitKey();
+#endif
+}
+
+TEST(xphoto_inpaint, smoke_FSR_FAST)  // fast smoke test, input doesn't fit well for tested algorithm
+{
+    test_inpainting(Size(128, 128), INPAINT_FSR_FAST, 30);
+}
+TEST(xphoto_inpaint, smoke_FSR_BEST)  // fast smoke test, input doesn't fit well for tested algorithm
+{
+    applyTestTag(CV_TEST_TAG_LONG);
+    test_inpainting(Size(128, 128), INPAINT_FSR_BEST, 30);
+}
+
+TEST(xphoto_inpaint, smoke_grayscale_FSR_FAST)  // fast smoke test, input doesn't fit well for tested algorithm
+{
+    test_inpainting(Size(128, 128), INPAINT_FSR_FAST, 30, IMREAD_GRAYSCALE);
+}
+TEST(xphoto_inpaint, smoke_grayscale_FSR_BEST)  // fast smoke test, input doesn't fit well for tested algorithm
+{
+    test_inpainting(Size(128, 128), INPAINT_FSR_BEST, 30, IMREAD_GRAYSCALE);
+}
+
+
+TEST(xphoto_inpaint, regression_FSR_FAST)
+{
+    test_inpainting(Size(512, 512), INPAINT_FSR_FAST, 39.5);
+}
+TEST(xphoto_inpaint, regression_FSR_BEST)
+{
+    applyTestTag(CV_TEST_TAG_VERYLONG);  // add --test_tag_enable=verylong to run this test
+    test_inpainting(Size(512, 512), INPAINT_FSR_BEST, 39.6);
+}
+
+
+}} // namespace