Merge pull request #13532 from Tytan:channel_exp_comp

Channels exposure compensators (#13532)

* feed compatible with single channel images

* Simplified BlockGainCompensator::apply

* ChannelsCompensator

* BlocksChannelsCompensator

* Make source level compatibility detector happy

modules/stitching/include/opencv2/stitching/detail/exposure_compensate.hpp

@@ -62,7 +62,7 @@ class CV_EXPORTS_W ExposureCompensator
 public:
     virtual ~ExposureCompensator() {}
 
-    enum { NO, GAIN, GAIN_BLOCKS };
+    enum { NO, GAIN, GAIN_BLOCKS, CHANNELS, CHANNELS_BLOCKS };
     CV_WRAP static Ptr<ExposureCompensator> createDefault(int type);
 
     /**
@@ -110,6 +110,7 @@ intensities, see @cite BL07 and @cite WJ10 for details.
 class CV_EXPORTS_W GainCompensator : public ExposureCompensator
 {
 public:
+    // This Constructor only exists to make source level compatibility detector happy
     CV_WRAP GainCompensator()
             : GainCompensator(1) {}
     CV_WRAP GainCompensator(int nr_feeds)
@@ -130,16 +131,13 @@ private:
     int nr_feeds_;
 };
 
-/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block
-intensities, see @cite UES01 for details.
+/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image
+intensities on each channel independantly.
  */
-class CV_EXPORTS_W BlocksGainCompensator : public ExposureCompensator
+class CV_EXPORTS_W ChannelsCompensator : public ExposureCompensator
 {
 public:
-    CV_WRAP BlocksGainCompensator(int bl_width = 32, int bl_height = 32)
-            : BlocksGainCompensator(bl_width, bl_height, 1) {}
-    CV_WRAP BlocksGainCompensator(int bl_width, int bl_height, int nr_feeds)
-            : bl_width_(bl_width), bl_height_(bl_height), nr_feeds_(nr_feeds) {setUpdateGain(true);}
+    CV_WRAP ChannelsCompensator(int nr_feeds=1) : nr_feeds_(nr_feeds) {}
     void feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
               const std::vector<std::pair<UMat,uchar> > &masks) CV_OVERRIDE;
     CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE;
@@ -147,12 +145,76 @@ public:
     CV_WRAP void setMatGains(std::vector<Mat>& umv) CV_OVERRIDE;
     CV_WRAP void setNrFeeds(int nr_feeds) { nr_feeds_ = nr_feeds; }
     CV_WRAP int getNrFeeds() { return nr_feeds_; }
+    std::vector<Scalar> gains() const { return gains_; }
 
 private:
+    std::vector<Scalar> gains_;
+    int nr_feeds_;
+};
+
+/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image blocks.
+ */
+class CV_EXPORTS_W BlocksCompensator : public ExposureCompensator
+{
+public:
+    BlocksCompensator(int bl_width=32, int bl_height=32, int nr_feeds=1)
+            : bl_width_(bl_width), bl_height_(bl_height), nr_feeds_(nr_feeds) {}
+    CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE;
+    CV_WRAP void getMatGains(CV_OUT std::vector<Mat>& umv) CV_OVERRIDE;
+    CV_WRAP void setMatGains(std::vector<Mat>& umv) CV_OVERRIDE;
+    CV_WRAP void setNrFeeds(int nr_feeds) { nr_feeds_ = nr_feeds; }
+    CV_WRAP int getNrFeeds() { return nr_feeds_; }
+
+protected:
+    template<class Compensator>
+    void feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+              const std::vector<std::pair<UMat,uchar> > &masks);
+
+private:
+    UMat getGainMap(const GainCompensator& compensator, int bl_idx, Size bl_per_img);
+    UMat getGainMap(const ChannelsCompensator& compensator, int bl_idx, Size bl_per_img);
+
     int bl_width_, bl_height_;
     std::vector<UMat> gain_maps_;
     int nr_feeds_;
 };
+
+/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block
+intensities, see @cite UES01 for details.
+ */
+class CV_EXPORTS_W BlocksGainCompensator : public BlocksCompensator
+{
+public:
+    // This Constructor only exists to make source level compatibility detector happy
+    CV_WRAP BlocksGainCompensator(int bl_width = 32, int bl_height = 32)
+            : BlocksGainCompensator(bl_width, bl_height, 1) {}
+    CV_WRAP BlocksGainCompensator(int bl_width, int bl_height, int nr_feeds)
+            : BlocksCompensator(bl_width, bl_height, nr_feeds) {setUpdateGain(true);}
+
+    void feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+              const std::vector<std::pair<UMat,uchar> > &masks) CV_OVERRIDE;
+
+    // This function only exists to make source level compatibility detector happy
+    CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE {
+        BlocksCompensator::apply(index, corner, image, mask); }
+    // This function only exists to make source level compatibility detector happy
+    CV_WRAP void getMatGains(CV_OUT std::vector<Mat>& umv) CV_OVERRIDE { BlocksCompensator::getMatGains(umv); }
+    // This function only exists to make source level compatibility detector happy
+    CV_WRAP void setMatGains(std::vector<Mat>& umv) CV_OVERRIDE { BlocksCompensator::setMatGains(umv); }
+};
+
+/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block
+on each channel.
+ */
+class CV_EXPORTS_W BlocksChannelsCompensator : public BlocksCompensator
+{
+public:
+    CV_WRAP BlocksChannelsCompensator(int bl_width=32, int bl_height=32, int nr_feeds=1)
+            : BlocksCompensator(bl_width, bl_height, nr_feeds) {setUpdateGain(true);}
+
+    void feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+              const std::vector<std::pair<UMat,uchar> > &masks) CV_OVERRIDE;
+};
 //! @}
 
 } // namespace detail

modules/stitching/perf/perf_stich.cpp

@@ -24,7 +24,7 @@ typedef TestBaseWithParam<tuple<string, int>> stitchExposureCompMultiFeed;
 #endif
 #define TEST_EXP_COMP_BS testing::Values(32, 16, 12, 10, 8)
 #define TEST_EXP_COMP_NR_FEED testing::Values(1, 2, 3, 4, 5)
-#define TEST_EXP_COMP_MODE testing::Values("gain", "blocks")
+#define TEST_EXP_COMP_MODE testing::Values("gain", "channels", "blocks_gain", "blocks_channels")
 #define AFFINE_DATASETS testing::Values("s", "budapest", "newspaper", "prague")
 
 PERF_TEST_P(stitch, a123, TEST_DETECTORS)
@@ -111,10 +111,14 @@ PERF_TEST_P(stitchExposureCompMultiFeed, a123, testing::Combine(TEST_EXP_COMP_MO
     declare.time(30 * 10).iterations(10);
 
     Ptr<detail::ExposureCompensator> exp_comp;
-    if (mode == "blocks")
-        exp_comp = makePtr<detail::BlocksGainCompensator>(block_size, block_size, nr_feeds);
-    else if (mode == "gain")
+    if (mode == "gain")
         exp_comp = makePtr<detail::GainCompensator>(nr_feeds);
+    else if (mode == "channels")
+        exp_comp = makePtr<detail::ChannelsCompensator>(nr_feeds);
+    else if (mode == "blocks_gain")
+        exp_comp = makePtr<detail::BlocksGainCompensator>(block_size, block_size, nr_feeds);
+    else if (mode == "blocks_channels")
+        exp_comp = makePtr<detail::BlocksChannelsCompensator>(block_size, block_size, nr_feeds);
 
     while(next())
     {

modules/stitching/src/exposure_compensate.cpp

@@ -56,8 +56,12 @@ Ptr<ExposureCompensator> ExposureCompensator::createDefault(int type)
         e = makePtr<NoExposureCompensator>();
     else if (type == GAIN)
         e = makePtr<GainCompensator>();
-    if (type == GAIN_BLOCKS)
+    else if (type == GAIN_BLOCKS)
         e = makePtr<BlocksGainCompensator>();
+    else if (type == CHANNELS)
+        e = makePtr<ChannelsCompensator>();
+    else if (type == CHANNELS_BLOCKS)
+        e = makePtr<BlocksChannelsCompensator>();
     if (e.get() != nullptr)
     {
         e->setUpdateGain(true);
@@ -114,12 +118,19 @@ void GainCompensator::singleFeed(const std::vector<Point> &corners, const std::v
 {
     CV_Assert(corners.size() == images.size() && images.size() == masks.size());
 
+    if (images.size() == 0)
+        return;
+
+    const int num_channels = images[0].channels();
+    CV_Assert(std::all_of(images.begin(), images.end(),
+        [num_channels](const UMat& image) { return image.channels() == num_channels; }));
+    CV_Assert(num_channels == 1 || num_channels == 3);
+
     const int num_images = static_cast<int>(images.size());
     Mat_<int> N(num_images, num_images); N.setTo(0);
     Mat_<double> I(num_images, num_images); I.setTo(0);
     Mat_<bool> skip(num_images, 1); skip.setTo(true);
 
-    //Rect dst_roi = resultRoi(corners, images);
     Mat subimg1, subimg2;
     Mat_<uchar> submask1, submask2, intersect;
 
@@ -154,14 +165,30 @@ void GainCompensator::singleFeed(const std::vector<Point> &corners, const std::v
                 double Isum1 = 0, Isum2 = 0;
                 for (int y = 0; y < roi.height; ++y)
                 {
-                    const Point3_<uchar>* r1 = subimg1.ptr<Point3_<uchar> >(y);
-                    const Point3_<uchar>* r2 = subimg2.ptr<Point3_<uchar> >(y);
+                    if (num_channels == 3)
+                    {
+                        const Vec<uchar, 3>* r1 = subimg1.ptr<Vec<uchar, 3> >(y);
+                        const Vec<uchar, 3>* r2 = subimg2.ptr<Vec<uchar, 3> >(y);
+                        for (int x = 0; x < roi.width; ++x)
+                        {
+                            if (intersect(y, x))
+                            {
+                                Isum1 += norm(r1[x]);
+                                Isum2 += norm(r2[x]);
+                            }
+                        }
+                    }
+                    else // if (num_channels == 1)
+                    {
+                        const uchar* r1 = subimg1.ptr<uchar>(y);
+                        const uchar* r2 = subimg2.ptr<uchar>(y);
                         for (int x = 0; x < roi.width; ++x)
                         {
                             if (intersect(y, x))
                             {
-                            Isum1 += std::sqrt(static_cast<double>(sqr(r1[x].x) + sqr(r1[x].y) + sqr(r1[x].z)));
-                            Isum2 += std::sqrt(static_cast<double>(sqr(r2[x].x) + sqr(r2[x].y) + sqr(r2[x].z)));
+                                Isum1 += r1[x];
+                                Isum2 += r2[x];
+                            }
                         }
                     }
                 }
@@ -268,8 +295,69 @@ void GainCompensator::setMatGains(std::vector<Mat>& umv)
     }
 }
 
+void ChannelsCompensator::feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+                               const std::vector<std::pair<UMat,uchar> > &masks)
+{
+    std::array<std::vector<UMat>, 3> images_channels;
 
-void BlocksGainCompensator::feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+    // Split channels of each input image
+    for (const UMat& image: images)
+    {
+        std::vector<UMat> image_channels;
+        image_channels.resize(3);
+        split(image, image_channels);
+
+        for (int i = 0; i < int(images_channels.size()); ++i)
+            images_channels[i].emplace_back(std::move(image_channels[i]));
+    }
+
+    // For each channel, feed the channel of each image in a GainCompensator
+    gains_.clear();
+    gains_.resize(images.size());
+    for (int c = 0; c < 3; ++c)
+    {
+        const std::vector<UMat>& channels = images_channels[c];
+
+        GainCompensator compensator(getNrFeeds());
+        compensator.feed(corners, channels, masks);
+
+        std::vector<double> gains = compensator.gains();
+        for (int i = 0; i < int(gains.size()); ++i)
+            gains_.at(i)[c] = gains[i];
+    }
+}
+
+void ChannelsCompensator::apply(int index, Point /*corner*/, InputOutputArray image, InputArray /*mask*/)
+{
+    CV_INSTRUMENT_REGION();
+
+    multiply(image, gains_.at(index), image);
+}
+
+void ChannelsCompensator::getMatGains(std::vector<Mat>& umv)
+{
+    umv.clear();
+    for (int i = 0; i < static_cast<int>(gains_.size()); ++i)
+    {
+        Mat m;
+        Mat(gains_[i]).copyTo(m);
+        umv.push_back(m);
+    }
+}
+
+void ChannelsCompensator::setMatGains(std::vector<Mat>& umv)
+{
+    for (int i = 0; i < static_cast<int>(umv.size()); i++)
+    {
+        Scalar s;
+        umv[i].copyTo(s);
+        gains_.push_back(s);
+    }
+}
+
+
+template<class Compensator>
+void BlocksCompensator::feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
                              const std::vector<std::pair<UMat,uchar> > &masks)
 {
     CV_Assert(corners.size() == images.size() && images.size() == masks.size());
@@ -305,11 +393,13 @@ void BlocksGainCompensator::feed(const std::vector<Point> &corners, const std::v
         }
     }
 
-    if (getUpdateGain())
+    if (getUpdateGain() || int(gain_maps_.size()) != num_images)
     {
-        GainCompensator compensator(nr_feeds_);
+        Compensator compensator;
+        compensator.setNrFeeds(getNrFeeds());
         compensator.feed(block_corners, block_images, block_masks);
-        std::vector<double> gains = compensator.gains();
+
+        gain_maps_.clear();
         gain_maps_.resize(num_images);
 
         Mat_<float> ker(1, 3);
@@ -319,50 +409,71 @@ void BlocksGainCompensator::feed(const std::vector<Point> &corners, const std::v
         for (int img_idx = 0; img_idx < num_images; ++img_idx)
         {
             Size bl_per_img = bl_per_imgs[img_idx];
-            gain_maps_[img_idx].create(bl_per_img, CV_32F);
+            UMat gain_map = getGainMap(compensator, bl_idx, bl_per_img);
+            bl_idx += bl_per_img.width*bl_per_img.height;
+
+            sepFilter2D(gain_map, gain_map, CV_32F, ker, ker);
+            sepFilter2D(gain_map, gain_map, CV_32F, ker, ker);
+
+            gain_maps_[img_idx] = gain_map;
+        }
+    }
+}
+
+UMat BlocksCompensator::getGainMap(const GainCompensator& compensator, int bl_idx, Size bl_per_img)
+{
+    std::vector<double> gains = compensator.gains();
+
+    UMat u_gain_map(bl_per_img, CV_32F);
+    Mat_<float> gain_map = u_gain_map.getMat(ACCESS_WRITE);
 
-            {
-                Mat_<float> gain_map = gain_maps_[img_idx].getMat(ACCESS_WRITE);
     for (int by = 0; by < bl_per_img.height; ++by)
         for (int bx = 0; bx < bl_per_img.width; ++bx, ++bl_idx)
             gain_map(by, bx) = static_cast<float>(gains[bl_idx]);
-            }
 
-            sepFilter2D(gain_maps_[img_idx], gain_maps_[img_idx], CV_32F, ker, ker);
-            sepFilter2D(gain_maps_[img_idx], gain_maps_[img_idx], CV_32F, ker, ker);
-        }
-    }
+    return u_gain_map;
 }
 
+UMat BlocksCompensator::getGainMap(const ChannelsCompensator& compensator, int bl_idx, Size bl_per_img)
+{
+    std::vector<Scalar> gains = compensator.gains();
+
+    UMat u_gain_map(bl_per_img, CV_32FC3);
+    Mat_<Vec3f> gain_map = u_gain_map.getMat(ACCESS_WRITE);
+
+    for (int by = 0; by < bl_per_img.height; ++by)
+        for (int bx = 0; bx < bl_per_img.width; ++bx, ++bl_idx)
+            for (int c = 0; c < 3; ++c)
+                gain_map(by, bx)[c] = static_cast<float>(gains[bl_idx][c]);
 
-void BlocksGainCompensator::apply(int index, Point /*corner*/, InputOutputArray _image, InputArray /*mask*/)
+    return u_gain_map;
+}
+
+void BlocksCompensator::apply(int index, Point /*corner*/, InputOutputArray _image, InputArray /*mask*/)
 {
     CV_INSTRUMENT_REGION();
 
     CV_Assert(_image.type() == CV_8UC3);
 
     UMat u_gain_map;
-    if (gain_maps_[index].size() == _image.size())
-        u_gain_map = gain_maps_[index];
+    if (gain_maps_.at(index).size() == _image.size())
+        u_gain_map = gain_maps_.at(index);
     else
-        resize(gain_maps_[index], u_gain_map, _image.size(), 0, 0, INTER_LINEAR);
+        resize(gain_maps_.at(index), u_gain_map, _image.size(), 0, 0, INTER_LINEAR);
 
-    Mat_<float> gain_map = u_gain_map.getMat(ACCESS_READ);
-    Mat image = _image.getMat();
-    for (int y = 0; y < image.rows; ++y)
-    {
-        const float* gain_row = gain_map.ptr<float>(y);
-        Point3_<uchar>* row = image.ptr<Point3_<uchar> >(y);
-        for (int x = 0; x < image.cols; ++x)
+    if (u_gain_map.channels() != 3)
     {
-            row[x].x = saturate_cast<uchar>(row[x].x * gain_row[x]);
-            row[x].y = saturate_cast<uchar>(row[x].y * gain_row[x]);
-            row[x].z = saturate_cast<uchar>(row[x].z * gain_row[x]);
-        }
+        std::vector<UMat> gains_channels;
+        gains_channels.push_back(u_gain_map);
+        gains_channels.push_back(u_gain_map);
+        gains_channels.push_back(u_gain_map);
+        merge(gains_channels, u_gain_map);
     }
+
+    multiply(_image, u_gain_map, _image, 1, _image.type());
 }
 
-void BlocksGainCompensator::getMatGains(std::vector<Mat>& umv)
+void BlocksCompensator::getMatGains(std::vector<Mat>& umv)
 {
     umv.clear();
     for (int i = 0; i < static_cast<int>(gain_maps_.size()); ++i)
@@ -372,7 +483,8 @@ void BlocksGainCompensator::getMatGains(std::vector<Mat>& umv)
         umv.push_back(m);
     }
 }
-void BlocksGainCompensator::setMatGains(std::vector<Mat>& umv)
+
+void BlocksCompensator::setMatGains(std::vector<Mat>& umv)
 {
     for (int i = 0; i < static_cast<int>(umv.size()); i++)
     {
@@ -382,6 +494,18 @@ void BlocksGainCompensator::setMatGains(std::vector<Mat>& umv)
     }
 }
 
+void BlocksGainCompensator::feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+                                 const std::vector<std::pair<UMat,uchar> > &masks)
+{
+    BlocksCompensator::feed<GainCompensator>(corners, images, masks);
+}
+
+void BlocksChannelsCompensator::feed(const std::vector<Point> &corners, const std::vector<UMat> &images,
+                                     const std::vector<std::pair<UMat,uchar> > &masks)
+{
+    BlocksCompensator::feed<ChannelsCompensator>(corners, images, masks);
+}
+
 
 } // namespace detail
 } // namespace cv